2024-09-14T03:18:38.661 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-14T03:18:38.685 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T03:18:38.777 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521 branch: main description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7904521' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'librbd python api tests ' name: teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: extra_system_packages: - python3-pytest flavor: default sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: main sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 8038 sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 sleep_before_teardown: 0 subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded targets: smithi110.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNUD/JXWyiy4JGK0XPe+VBlCXy993k7fnPjVsEx+ierx/1SRRfc62zz2zXsP+5co8gLhOt4Y2gph7ncRNQdF8sg= smithi195.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOqTcgtOxGEQKwAMJXUeNF5+OGOMOMedBa9tgg6MAHEHbAUZqRyZhkO6m6fIinWZOBfDIeYEXVAsp8W9UPiecCY= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install nvmetcli nvme-cli -y - install: branch: quincy exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing quincy cephadm ...' - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - print: '**** done end installing quincy cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry quincy...' - workunit: clients: client.0: - test_telemetry_quincy.sh - print: '**** done end telemetry quincy...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_quincy_x.sh - print: '**** done end telemetry x...' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-14_03:08:15 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: quincy clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: quincy clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: quincy clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_api.yaml' - print: '**** done start test_rbd_python.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd_python.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_python.yaml' 2024-09-14T03:18:38.777 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa; will attempt to use it 2024-09-14T03:18:38.778 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks 2024-09-14T03:18:38.778 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-14T03:18:38.780 INFO:teuthology.task.internal:Checking packages... 2024-09-14T03:18:38.802 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8' 2024-09-14T03:18:38.802 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:18:38.802 INFO:teuthology.packaging:ref: None 2024-09-14T03:18:38.802 INFO:teuthology.packaging:tag: None 2024-09-14T03:18:38.802 INFO:teuthology.packaging:branch: main 2024-09-14T03:18:38.802 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:18:38.802 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-14T03:18:39.015 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4926.g1c941a66 2024-09-14T03:18:39.016 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-14T03:18:39.024 INFO:teuthology.task.internal:no buildpackages task found 2024-09-14T03:18:39.024 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-14T03:18:39.072 INFO:teuthology.task.internal:Saving configuration 2024-09-14T03:18:39.091 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-14T03:18:39.099 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-14T03:18:39.122 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi110.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:13:49.961472', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNUD/JXWyiy4JGK0XPe+VBlCXy993k7fnPjVsEx+ierx/1SRRfc62zz2zXsP+5co8gLhOt4Y2gph7ncRNQdF8sg='} 2024-09-14T03:18:39.147 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi195.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:13:49.963250', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOqTcgtOxGEQKwAMJXUeNF5+OGOMOMedBa9tgg6MAHEHbAUZqRyZhkO6m6fIinWZOBfDIeYEXVAsp8W9UPiecCY='} 2024-09-14T03:18:39.147 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-14T03:18:39.156 INFO:teuthology.task.internal:roles: ubuntu@smithi110.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-14T03:18:39.156 INFO:teuthology.task.internal:roles: ubuntu@smithi195.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-14T03:18:39.156 INFO:teuthology.run_tasks:Running task console_log... 2024-09-14T03:18:39.240 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f451339fac0>, signals=[15]) 2024-09-14T03:18:39.241 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-14T03:18:39.247 INFO:teuthology.task.internal:Opening connections... 2024-09-14T03:18:39.247 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi110.front.sepia.ceph.com 2024-09-14T03:18:39.248 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:18:39.347 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi195.front.sepia.ceph.com 2024-09-14T03:18:39.348 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:18:39.428 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-14T03:18:39.437 DEBUG:teuthology.orchestra.run.smithi110:> uname -m 2024-09-14T03:18:39.455 INFO:teuthology.orchestra.run.smithi110.stdout:x86_64 2024-09-14T03:18:39.456 DEBUG:teuthology.orchestra.run.smithi110:> cat /etc/os-release 2024-09-14T03:18:39.514 INFO:teuthology.orchestra.run.smithi110.stdout:NAME="CentOS Stream" 2024-09-14T03:18:39.514 INFO:teuthology.orchestra.run.smithi110.stdout:VERSION="9" 2024-09-14T03:18:39.514 INFO:teuthology.orchestra.run.smithi110.stdout:ID="centos" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:VERSION_ID="9" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:ANSI_COLOR="0;31" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:18:39.515 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:18:39.516 INFO:teuthology.lock.ops:Updating smithi110.front.sepia.ceph.com on lock server 2024-09-14T03:18:39.582 DEBUG:teuthology.orchestra.run.smithi195:> uname -m 2024-09-14T03:18:39.601 INFO:teuthology.orchestra.run.smithi195.stdout:x86_64 2024-09-14T03:18:39.602 DEBUG:teuthology.orchestra.run.smithi195:> cat /etc/os-release 2024-09-14T03:18:39.659 INFO:teuthology.orchestra.run.smithi195.stdout:NAME="CentOS Stream" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION="9" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:ID="centos" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION_ID="9" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:ANSI_COLOR="0;31" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:18:39.660 INFO:teuthology.orchestra.run.smithi195.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:18:39.661 INFO:teuthology.orchestra.run.smithi195.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:18:39.661 INFO:teuthology.orchestra.run.smithi195.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:18:39.661 INFO:teuthology.orchestra.run.smithi195.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:18:39.661 INFO:teuthology.lock.ops:Updating smithi195.front.sepia.ceph.com on lock server 2024-09-14T03:18:39.689 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-14T03:18:39.698 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-14T03:18:39.737 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-14T03:18:39.737 DEBUG:teuthology.orchestra.run.smithi110:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:18:39.740 DEBUG:teuthology.orchestra.run.smithi195:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:18:39.757 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-14T03:18:39.776 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-14T03:18:39.776 DEBUG:teuthology.orchestra.run.smithi110:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:18:39.798 DEBUG:teuthology.orchestra.run.smithi195:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:18:39.825 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-14T03:18:39.877 INFO:teuthology.run_tasks:Running task kernel... 2024-09-14T03:18:39.897 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-14T03:18:39.897 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-14T03:18:39.897 DEBUG:teuthology.orchestra.run.smithi110:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:18:39.898 DEBUG:teuthology.orchestra.run.smithi195:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:18:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:18:39.915 DEBUG:teuthology.orchestra.run.smithi110:> uname -r 2024-09-14T03:18:39.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:18:39.916 DEBUG:teuthology.orchestra.run.smithi195:> uname -r 2024-09-14T03:18:39.972 INFO:teuthology.orchestra.run.smithi110.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:18:39.972 INFO:teuthology.task.kernel:Running kernel on smithi110: 5.14.0-503.el9.x86_64 2024-09-14T03:18:39.973 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum install -y kernel 2024-09-14T03:18:39.974 INFO:teuthology.orchestra.run.smithi195.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:18:39.975 INFO:teuthology.task.kernel:Running kernel on smithi195: 5.14.0-503.el9.x86_64 2024-09-14T03:18:39.975 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum install -y kernel 2024-09-14T03:18:42.320 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.2 MB 00:00 2024-09-14T03:18:42.560 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.2 MB 00:00 2024-09-14T03:18:46.013 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - AppStream 7.9 MB/s | 20 MB 00:02 2024-09-14T03:18:48.193 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - AppStream 4.5 MB/s | 20 MB 00:04 2024-09-14T03:18:53.760 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 6.5 MB 00:01 2024-09-14T03:18:53.955 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - CRB 1.6 MB/s | 6.5 MB 00:04 2024-09-14T03:18:55.236 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - Extras packages 65 kB/s | 19 kB 00:00 2024-09-14T03:18:55.605 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - Extras packages 64 kB/s | 19 kB 00:00 2024-09-14T03:18:56.415 INFO:teuthology.orchestra.run.smithi195.stdout:Extra Packages for Enterprise Linux 21 MB/s | 23 MB 00:01 2024-09-14T03:18:56.468 INFO:teuthology.orchestra.run.smithi110.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-14T03:19:01.829 INFO:teuthology.orchestra.run.smithi195.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-09-14T03:19:01.840 INFO:teuthology.orchestra.run.smithi110.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-14T03:19:03.864 INFO:teuthology.orchestra.run.smithi110.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:19:03.864 INFO:teuthology.orchestra.run.smithi110.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:19:03.869 INFO:teuthology.orchestra.run.smithi195.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:19:03.869 INFO:teuthology.orchestra.run.smithi195.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:19:03.949 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2024-09-14T03:19:03.957 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:19:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2024-09-14T03:19:03.957 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:19:03.957 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2024-09-14T03:19:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout:Install 4 Packages 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 79 M 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 126 M 2024-09-14T03:19:03.958 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2024-09-14T03:19:03.959 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:19:03.967 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:19:03.968 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-09-14T03:19:03.968 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:19:03.968 INFO:teuthology.orchestra.run.smithi195.stdout:Install 4 Packages 2024-09-14T03:19:03.968 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:19:03.968 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 79 M 2024-09-14T03:19:03.968 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 126 M 2024-09-14T03:19:03.969 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-09-14T03:19:04.158 INFO:teuthology.orchestra.run.smithi110.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 265 kB/s | 23 kB 00:00 2024-09-14T03:19:04.517 INFO:teuthology.orchestra.run.smithi110.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 35 MB/s | 16 MB 00:00 2024-09-14T03:19:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 43 kB/s | 23 kB 00:00 2024-09-14T03:19:04.926 INFO:teuthology.orchestra.run.smithi110.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 40 MB/s | 35 MB 00:00 2024-09-14T03:19:05.360 INFO:teuthology.orchestra.run.smithi110.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-14T03:19:05.360 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:19:05.360 INFO:teuthology.orchestra.run.smithi110.stdout:Total 56 MB/s | 79 MB 00:01 2024-09-14T03:19:05.498 INFO:teuthology.orchestra.run.smithi195.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-14T03:19:06.035 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2024-09-14T03:19:06.173 INFO:teuthology.orchestra.run.smithi195.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 17 MB/s | 35 MB 00:02 2024-09-14T03:19:06.399 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2024-09-14T03:19:06.399 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2024-09-14T03:19:07.115 INFO:teuthology.orchestra.run.smithi195.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 11 MB/s | 29 MB 00:02 2024-09-14T03:19:07.115 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:19:07.115 INFO:teuthology.orchestra.run.smithi195.stdout:Total 25 MB/s | 79 MB 00:03 2024-09-14T03:19:07.347 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2024-09-14T03:19:07.347 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2024-09-14T03:19:07.803 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-09-14T03:19:08.150 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-09-14T03:19:08.150 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-09-14T03:19:09.119 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-09-14T03:19:09.119 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-09-14T03:19:09.784 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2024-09-14T03:19:10.460 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:19:10.553 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:19:11.528 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-09-14T03:19:12.172 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:19:12.248 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:19:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:19:13.480 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:19:15.042 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:19:15.169 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:19:18.302 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:19:18.397 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:19:19.980 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:19:20.094 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:19:23.225 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:19:24.951 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:01.347 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:01.393 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:01.733 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:01.734 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:20:01.734 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:20:01.734 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:20:02.604 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:02.604 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:20:02.604 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2024-09-14T03:20:02.604 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:20:02.604 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:20:02.605 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:20:02.605 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:20:02.605 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:20:02.605 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2024-09-14T03:20:02.845 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:02.884 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:02.978 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:20:03.060 INFO:teuthology.orchestra.run.smithi110.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:20:03.061 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:20:03.061 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:20:03.062 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:20:03.062 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:20:03.062 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-14T03:20:03.062 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi110.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:20:03.062 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:20:03.122 INFO:teuthology.orchestra.run.smithi110.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:20:03.122 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:20:03.122 DEBUG:teuthology.orchestra.run.smithi110:> sudo rpm -qi grub2-tools 2024-09-14T03:20:03.154 INFO:teuthology.orchestra.run.smithi110.stdout:Name : grub2-tools 2024-09-14T03:20:03.154 INFO:teuthology.orchestra.run.smithi110.stdout:Epoch : 1 2024-09-14T03:20:03.154 INFO:teuthology.orchestra.run.smithi110.stdout:Version : 2.06 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Release : 82.el9 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Architecture: x86_64 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Group : Unspecified 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Size : 8274670 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:License : GPLv3+ 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Packager : builder@centos.org 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Vendor : CentOS 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:20:03.155 INFO:teuthology.orchestra.run.smithi110.stdout:Summary : Support tools for GRUB. 2024-09-14T03:20:03.156 INFO:teuthology.orchestra.run.smithi110.stdout:Description : 2024-09-14T03:20:03.156 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:20:03.156 INFO:teuthology.orchestra.run.smithi110.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:20:03.156 INFO:teuthology.orchestra.run.smithi110.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:20:03.156 INFO:teuthology.orchestra.run.smithi110.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:20:03.156 INFO:teuthology.orchestra.run.smithi110.stdout:hardware devices. 2024-09-14T03:20:03.156 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:20:03.156 INFO:teuthology.orchestra.run.smithi110.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:20:03.157 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:20:03.157 INFO:teuthology.task.kernel:Updating grub on smithi110 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:20:03.157 DEBUG:teuthology.orchestra.run.smithi110:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:20:03.232 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:03.232 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:20:03.233 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:20:03.233 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:20:03.864 INFO:teuthology.orchestra.run.smithi110.stderr:Generating grub configuration file ... 2024-09-14T03:20:04.063 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:20:04.063 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:20:04.063 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-09-14T03:20:04.063 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:20:04.063 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:20:04.063 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:20:04.064 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:20:04.064 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:20:04.064 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-09-14T03:20:04.449 DEBUG:teuthology.orchestra.run.smithi195:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:20:04.535 INFO:teuthology.orchestra.run.smithi195.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:20:04.536 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:20:04.536 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:20:04.536 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:20:04.536 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:20:04.536 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-14T03:20:04.536 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi195.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:20:04.536 DEBUG:teuthology.orchestra.run.smithi195:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:20:04.595 INFO:teuthology.orchestra.run.smithi195.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:20:04.595 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:20:04.595 DEBUG:teuthology.orchestra.run.smithi195:> sudo rpm -qi grub2-tools 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:Name : grub2-tools 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:Epoch : 1 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:Version : 2.06 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:Release : 82.el9 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:Architecture: x86_64 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:Group : Unspecified 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:Size : 8274670 2024-09-14T03:20:04.630 INFO:teuthology.orchestra.run.smithi195.stdout:License : GPLv3+ 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:Packager : builder@centos.org 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:Vendor : CentOS 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:Summary : Support tools for GRUB. 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:Description : 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:20:04.631 INFO:teuthology.orchestra.run.smithi195.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:20:04.632 INFO:teuthology.orchestra.run.smithi195.stdout:hardware devices. 2024-09-14T03:20:04.632 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:20:04.632 INFO:teuthology.orchestra.run.smithi195.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:20:04.633 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:20:04.633 INFO:teuthology.task.kernel:Updating grub on smithi195 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:20:04.633 DEBUG:teuthology.orchestra.run.smithi195:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:20:05.124 INFO:teuthology.orchestra.run.smithi110.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:20:05.154 INFO:teuthology.orchestra.run.smithi110.stderr:done 2024-09-14T03:20:05.164 DEBUG:teuthology.orchestra.run.smithi110:> mktemp 2024-09-14T03:20:05.180 INFO:teuthology.orchestra.run.smithi110.stdout:/tmp/tmp.RdO31fZTRi 2024-09-14T03:20:05.180 DEBUG:teuthology.orchestra.run.smithi110:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.RdO31fZTRi 2024-09-14T03:20:05.247 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 0666 /tmp/tmp.RdO31fZTRi 2024-09-14T03:20:05.352 INFO:teuthology.orchestra.run.smithi195.stderr:Generating grub configuration file ... 2024-09-14T03:20:05.404 DEBUG:teuthology.orchestra.remote:smithi110:/tmp/tmp.RdO31fZTRi is 6KB 2024-09-14T03:20:05.425 DEBUG:teuthology.orchestra.run.smithi110:> rm -fr /tmp/tmp.RdO31fZTRi 2024-09-14T03:20:05.441 DEBUG:teuthology.orchestra.run.smithi110:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:20:05.503 INFO:teuthology.orchestra.run.smithi110.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:20:05.504 INFO:teuthology.orchestra.run.smithi110.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:20:05.504 INFO:teuthology.orchestra.run.smithi110.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:20:05.504 INFO:teuthology.orchestra.run.smithi110.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:20:05.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:20:05.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo shutdown -r now 2024-09-14T03:20:06.686 INFO:teuthology.orchestra.run.smithi195.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:20:06.731 INFO:teuthology.orchestra.run.smithi195.stderr:done 2024-09-14T03:20:06.733 DEBUG:teuthology.orchestra.run.smithi195:> mktemp 2024-09-14T03:20:06.748 INFO:teuthology.orchestra.run.smithi195.stdout:/tmp/tmp.vnLyHT6V90 2024-09-14T03:20:06.748 DEBUG:teuthology.orchestra.run.smithi195:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.vnLyHT6V90 2024-09-14T03:20:06.826 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 0666 /tmp/tmp.vnLyHT6V90 2024-09-14T03:20:06.963 DEBUG:teuthology.orchestra.remote:smithi195:/tmp/tmp.vnLyHT6V90 is 6KB 2024-09-14T03:20:07.014 DEBUG:teuthology.orchestra.run.smithi195:> rm -fr /tmp/tmp.vnLyHT6V90 2024-09-14T03:20:07.030 DEBUG:teuthology.orchestra.run.smithi195:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:20:07.092 INFO:teuthology.orchestra.run.smithi195.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:20:07.093 INFO:teuthology.orchestra.run.smithi195.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:20:07.093 INFO:teuthology.orchestra.run.smithi195.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:20:07.093 INFO:teuthology.orchestra.run.smithi195.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:20:07.094 DEBUG:teuthology.orchestra.run.smithi195:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:20:07.246 DEBUG:teuthology.orchestra.run.smithi195:> sudo shutdown -r now 2024-09-14T03:20:35.693 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-14T03:20:35.693 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-14T03:20:35.694 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:20:37.251 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-14T03:20:37.251 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-14T03:20:37.252 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:20:55.617 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.195 2024-09-14T03:21:04.626 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-14T03:21:04.628 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:21:35.725 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T03:21:44.733 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-14T03:21:44.734 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:21:45.071 DEBUG:teuthology.orchestra.run.smithi110:> true 2024-09-14T03:21:45.402 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-14T03:21:45.402 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:21:45.403 DEBUG:teuthology.orchestra.run.smithi110:> uname -r 2024-09-14T03:21:45.418 INFO:teuthology.orchestra.run.smithi110.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:21:45.418 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:21:45.418 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:21:45.418 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:21:46.420 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-14T03:21:46.420 DEBUG:teuthology.orchestra.run.smithi110:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:21:46.528 INFO:teuthology.orchestra.run.smithi110.stdout:ttyS1 2024-09-14T03:21:46.558 DEBUG:teuthology.parallel:result is None 2024-09-14T03:22:04.630 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T03:22:16.632 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-14T03:22:16.633 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:22:16.918 DEBUG:teuthology.orchestra.run.smithi195:> true 2024-09-14T03:22:17.282 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-14T03:22:17.282 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:22:17.282 DEBUG:teuthology.orchestra.run.smithi195:> uname -r 2024-09-14T03:22:17.298 INFO:teuthology.orchestra.run.smithi195.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:22:17.298 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:22:17.298 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:22:17.298 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:22:18.299 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-14T03:22:18.300 DEBUG:teuthology.orchestra.run.smithi195:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:22:18.368 INFO:teuthology.orchestra.run.smithi195.stdout:ttyS1 2024-09-14T03:22:18.391 DEBUG:teuthology.parallel:result is None 2024-09-14T03:22:18.391 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-14T03:22:18.397 INFO:teuthology.task.internal:Creating test directory... 2024-09-14T03:22:18.398 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:22:18.400 DEBUG:teuthology.orchestra.run.smithi195:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:22:18.418 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-14T03:22:18.432 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-14T03:22:18.464 INFO:teuthology.task.internal:Creating archive directory... 2024-09-14T03:22:18.464 DEBUG:teuthology.orchestra.run.smithi110:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:22:18.467 DEBUG:teuthology.orchestra.run.smithi195:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:22:18.499 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-14T03:22:18.506 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-14T03:22:18.506 DEBUG:teuthology.orchestra.run.smithi110:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:22:18.535 DEBUG:teuthology.orchestra.run.smithi195:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:22:18.570 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:22:18.573 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:22:18.581 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:22:18.585 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:22:18.587 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-14T03:22:18.594 INFO:teuthology.task.internal:Configuring sudo... 2024-09-14T03:22:18.594 DEBUG:teuthology.orchestra.run.smithi110:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:22:18.626 DEBUG:teuthology.orchestra.run.smithi195:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:22:18.655 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-14T03:22:18.664 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-14T03:22:18.664 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:22:18.694 DEBUG:teuthology.orchestra.run.smithi195:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:22:18.711 DEBUG:teuthology.orchestra.run.smithi110:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:22:18.793 DEBUG:teuthology.orchestra.run.smithi110:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:22:18.891 DEBUG:teuthology.orchestra.run.smithi110:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:22:18.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:22:18.976 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:22:18.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:22:19.039 DEBUG:teuthology.orchestra.run.smithi195:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:22:19.083 DEBUG:teuthology.orchestra.run.smithi195:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:22:19.135 DEBUG:teuthology.orchestra.run.smithi195:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:22:19.219 DEBUG:teuthology.orchestra.run.smithi195:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:22:19.261 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:22:19.262 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:22:19.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo service rsyslog restart 2024-09-14T03:22:19.330 DEBUG:teuthology.orchestra.run.smithi195:> sudo service rsyslog restart 2024-09-14T03:22:19.389 INFO:teuthology.orchestra.run.smithi110.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:22:19.414 INFO:teuthology.orchestra.run.smithi195.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:22:19.745 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-14T03:22:19.752 INFO:teuthology.task.internal:Starting timer... 2024-09-14T03:22:19.753 INFO:teuthology.run_tasks:Running task pcp... 2024-09-14T03:22:19.786 INFO:teuthology.run_tasks:Running task selinux... 2024-09-14T03:22:19.818 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-09-14T03:22:19.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo service auditd rotate 2024-09-14T03:22:19.899 INFO:teuthology.orchestra.run.smithi110.stdout:Rotating logs: 2024-09-14T03:22:19.900 DEBUG:teuthology.orchestra.run.smithi195:> sudo service auditd rotate 2024-09-14T03:22:19.971 INFO:teuthology.orchestra.run.smithi195.stdout:Rotating logs: 2024-09-14T03:22:19.973 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-14T03:22:19.973 DEBUG:teuthology.orchestra.run.smithi110:> /usr/sbin/getenforce 2024-09-14T03:22:19.991 INFO:teuthology.orchestra.run.smithi110.stdout:Permissive 2024-09-14T03:22:19.992 DEBUG:teuthology.orchestra.run.smithi195:> /usr/sbin/getenforce 2024-09-14T03:22:20.021 INFO:teuthology.orchestra.run.smithi195.stdout:Permissive 2024-09-14T03:22:20.021 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi110.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi195.front.sepia.ceph.com': 'permissive'} 2024-09-14T03:22:20.022 DEBUG:teuthology.orchestra.run.smithi110:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-14T03:22:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:22:20.057 DEBUG:teuthology.orchestra.run.smithi195:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-14T03:22:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:22:20.089 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-14T03:22:20.089 DEBUG:teuthology.orchestra.run.smithi110:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:22:20.128 DEBUG:teuthology.orchestra.run.smithi195:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:22:20.170 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-14T03:22:20.180 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-14T03:22:20.192 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-14T03:22:20.193 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi110.front.sepia.ceph.com,smithi195.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-14T03:26:03.412 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi110.front.sepia.ceph.com'), Remote(name='ubuntu@smithi195.front.sepia.ceph.com')] 2024-09-14T03:26:03.413 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-14T03:26:03.414 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:26:03.490 DEBUG:teuthology.orchestra.run.smithi110:> true 2024-09-14T03:26:03.567 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-14T03:26:03.567 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-14T03:26:03.568 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:26:03.643 DEBUG:teuthology.orchestra.run.smithi195:> true 2024-09-14T03:26:03.728 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-14T03:26:03.729 INFO:teuthology.run_tasks:Running task clock... 2024-09-14T03:26:03.738 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-14T03:26:03.738 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:26:03.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:26:03.741 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:26:03.741 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:26:03.768 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:26:03.782 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:26:03.798 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:26:03.806 INFO:teuthology.orchestra.run.smithi110.stderr:sudo: ntpd: command not found 2024-09-14T03:26:03.813 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:26:03.817 INFO:teuthology.orchestra.run.smithi110.stdout:506 Cannot talk to daemon 2024-09-14T03:26:03.831 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:26:03.838 INFO:teuthology.orchestra.run.smithi195.stderr:sudo: ntpd: command not found 2024-09-14T03:26:03.845 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:26:03.849 INFO:teuthology.orchestra.run.smithi195.stdout:506 Cannot talk to daemon 2024-09-14T03:26:03.863 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:26:03.877 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:26:03.896 INFO:teuthology.orchestra.run.smithi110.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:26:03.899 INFO:teuthology.orchestra.run.smithi110.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:26:03.900 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================== 2024-09-14T03:26:03.900 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:26:03.900 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:26:03.900 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:26:03.900 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:26:03.929 INFO:teuthology.orchestra.run.smithi195.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:26:03.932 INFO:teuthology.orchestra.run.smithi195.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:26:03.932 INFO:teuthology.orchestra.run.smithi195.stdout:=============================================================================== 2024-09-14T03:26:03.933 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:26:03.933 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:26:03.933 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:26:03.933 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:26:03.934 INFO:teuthology.run_tasks:Running task pexec... 2024-09-14T03:26:03.943 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-14T03:26:03.944 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi110.front.sepia.ceph.com 2024-09-14T03:26:03.944 DEBUG:teuthology.orchestra.run.smithi110:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-14T03:26:03.944 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi195.front.sepia.ceph.com 2024-09-14T03:26:03.944 DEBUG:teuthology.orchestra.run.smithi195:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-14T03:26:04.295 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout:Removing: 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout:Removing unused dependencies: 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout:Remove 2 Packages 2024-09-14T03:26:04.310 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:04.311 INFO:teuthology.orchestra.run.smithi110.stdout:Freed space: 5.5 M 2024-09-14T03:26:04.311 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2024-09-14T03:26:04.314 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2024-09-14T03:26:04.314 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2024-09-14T03:26:04.326 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-09-14T03:26:04.328 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:26:04.328 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-09-14T03:26:04.328 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:26:04.328 INFO:teuthology.orchestra.run.smithi195.stdout:Removing: 2024-09-14T03:26:04.328 INFO:teuthology.orchestra.run.smithi195.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-14T03:26:04.328 INFO:teuthology.orchestra.run.smithi195.stdout:Removing unused dependencies: 2024-09-14T03:26:04.329 INFO:teuthology.orchestra.run.smithi195.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-14T03:26:04.329 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:04.329 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-09-14T03:26:04.329 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:26:04.329 INFO:teuthology.orchestra.run.smithi195.stdout:Remove 2 Packages 2024-09-14T03:26:04.329 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:04.329 INFO:teuthology.orchestra.run.smithi195.stdout:Freed space: 5.5 M 2024-09-14T03:26:04.329 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-09-14T03:26:04.333 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-09-14T03:26:04.333 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-09-14T03:26:04.397 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-09-14T03:26:04.398 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-09-14T03:26:04.408 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2024-09-14T03:26:04.408 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2024-09-14T03:26:04.545 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2024-09-14T03:26:04.560 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-09-14T03:26:04.625 INFO:teuthology.orchestra.run.smithi110.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-14T03:26:04.642 INFO:teuthology.orchestra.run.smithi195.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-14T03:26:04.680 INFO:teuthology.orchestra.run.smithi110.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:26:04.702 INFO:teuthology.orchestra.run.smithi195.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:26:05.201 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:26:05.202 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-14T03:26:05.203 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:26:05.203 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-14T03:26:05.448 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-14T03:26:05.448 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:05.448 INFO:teuthology.orchestra.run.smithi110.stdout:Removed: 2024-09-14T03:26:05.448 INFO:teuthology.orchestra.run.smithi110.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:26:05.448 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:05.449 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2024-09-14T03:26:05.449 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-14T03:26:05.449 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:05.449 INFO:teuthology.orchestra.run.smithi195.stdout:Removed: 2024-09-14T03:26:05.449 INFO:teuthology.orchestra.run.smithi195.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:26:05.449 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:05.449 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-09-14T03:26:06.260 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:48 ago on Sat 14 Sep 2024 03:25:18 AM UTC. 2024-09-14T03:26:06.271 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:48 ago on Sat 14 Sep 2024 03:25:18 AM UTC. 2024-09-14T03:26:06.396 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-09-14T03:26:06.396 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:06.397 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-09-14T03:26:06.398 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-14T03:26:06.398 INFO:teuthology.orchestra.run.smithi195.stdout:Install 6 Packages 2024-09-14T03:26:06.398 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:06.406 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:06.407 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2024-09-14T03:26:06.408 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-14T03:26:06.408 INFO:teuthology.orchestra.run.smithi110.stdout:Install 6 Packages 2024-09-14T03:26:06.408 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:06.409 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 2.0 M 2024-09-14T03:26:06.409 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 9.2 M 2024-09-14T03:26:06.409 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-09-14T03:26:06.415 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 2.0 M 2024-09-14T03:26:06.415 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 9.2 M 2024-09-14T03:26:06.415 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2024-09-14T03:26:06.804 INFO:teuthology.orchestra.run.smithi195.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 397 kB/s | 44 kB 00:00 2024-09-14T03:26:06.829 INFO:teuthology.orchestra.run.smithi195.stdout:(2/6): libnvme-1.9-3.el9.x86_64.rpm 772 kB/s | 106 kB 00:00 2024-09-14T03:26:06.888 INFO:teuthology.orchestra.run.smithi195.stdout:(3/6): nvme-cli-2.9.1-6.el9.x86_64.rpm 4.5 MB/s | 906 kB 00:00 2024-09-14T03:26:06.913 INFO:teuthology.orchestra.run.smithi195.stdout:(4/6): python3-configshell-1.1.30-1.el9.noarch. 658 kB/s | 72 kB 00:00 2024-09-14T03:26:06.938 INFO:teuthology.orchestra.run.smithi195.stdout:(5/6): python3-kmod-0.9-32.el9.x86_64.rpm 773 kB/s | 84 kB 00:00 2024-09-14T03:26:07.022 INFO:teuthology.orchestra.run.smithi195.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 6.1 MB/s | 837 kB 00:00 2024-09-14T03:26:07.022 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:26:07.022 INFO:teuthology.orchestra.run.smithi195.stdout:Total 3.3 MB/s | 2.0 MB 00:00 2024-09-14T03:26:07.096 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-09-14T03:26:07.110 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-09-14T03:26:07.110 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-09-14T03:26:07.269 INFO:teuthology.orchestra.run.smithi110.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 77 kB/s | 44 kB 00:00 2024-09-14T03:26:07.388 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-09-14T03:26:07.389 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-09-14T03:26:07.436 INFO:teuthology.orchestra.run.smithi110.stdout:(2/6): libnvme-1.9-3.el9.x86_64.rpm 143 kB/s | 106 kB 00:00 2024-09-14T03:26:07.813 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-09-14T03:26:07.907 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/6 2024-09-14T03:26:07.942 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/6 2024-09-14T03:26:07.970 INFO:teuthology.orchestra.run.smithi110.stdout:(3/6): python3-configshell-1.1.30-1.el9.noarch. 102 kB/s | 72 kB 00:00 2024-09-14T03:26:07.970 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/6 2024-09-14T03:26:08.070 INFO:teuthology.orchestra.run.smithi110.stdout:(4/6): python3-kmod-0.9-32.el9.x86_64.rpm 133 kB/s | 84 kB 00:00 2024-09-14T03:26:08.194 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/6 2024-09-14T03:26:08.195 INFO:teuthology.orchestra.run.smithi110.stdout:(5/6): nvme-cli-2.9.1-6.el9.x86_64.rpm 603 kB/s | 906 kB 00:01 2024-09-14T03:26:08.235 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-14T03:26:08.678 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-14T03:26:08.707 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/6 2024-09-14T03:26:09.546 INFO:teuthology.orchestra.run.smithi110.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 532 kB/s | 837 kB 00:01 2024-09-14T03:26:09.547 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:26:09.547 INFO:teuthology.orchestra.run.smithi110.stdout:Total 655 kB/s | 2.0 MB 00:03 2024-09-14T03:26:09.625 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2024-09-14T03:26:09.639 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2024-09-14T03:26:09.640 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2024-09-14T03:26:09.716 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/6 2024-09-14T03:26:09.717 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/6 2024-09-14T03:26:09.717 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/6 2024-09-14T03:26:09.717 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/6 2024-09-14T03:26:09.717 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/6 2024-09-14T03:26:09.717 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/6 2024-09-14T03:26:09.942 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2024-09-14T03:26:09.943 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2024-09-14T03:26:10.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/6 2024-09-14T03:26:10.091 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:10.091 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-09-14T03:26:10.091 INFO:teuthology.orchestra.run.smithi195.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:26:10.091 INFO:teuthology.orchestra.run.smithi195.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-14T03:26:10.091 INFO:teuthology.orchestra.run.smithi195.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-14T03:26:10.091 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:10.091 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-09-14T03:26:10.356 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2024-09-14T03:26:10.376 DEBUG:teuthology.parallel:result is None 2024-09-14T03:26:10.440 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/6 2024-09-14T03:26:10.483 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/6 2024-09-14T03:26:10.520 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/6 2024-09-14T03:26:10.724 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/6 2024-09-14T03:26:10.759 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-14T03:26:11.192 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-14T03:26:11.221 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/6 2024-09-14T03:26:12.187 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/6 2024-09-14T03:26:12.187 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/6 2024-09-14T03:26:12.187 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/6 2024-09-14T03:26:12.187 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/6 2024-09-14T03:26:12.187 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/6 2024-09-14T03:26:12.187 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/6 2024-09-14T03:26:12.536 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/6 2024-09-14T03:26:12.536 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:12.536 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2024-09-14T03:26:12.536 INFO:teuthology.orchestra.run.smithi110.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:26:12.537 INFO:teuthology.orchestra.run.smithi110.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-14T03:26:12.537 INFO:teuthology.orchestra.run.smithi110.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-14T03:26:12.537 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:12.537 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2024-09-14T03:26:12.881 DEBUG:teuthology.parallel:result is None 2024-09-14T03:26:12.881 INFO:teuthology.run_tasks:Running task install... 2024-09-14T03:26:12.891 DEBUG:teuthology.task.install:project ceph 2024-09-14T03:26:12.891 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}} 2024-09-14T03:26:12.891 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:26:12.891 INFO:teuthology.task.install:Using flavor: default 2024-09-14T03:26:12.897 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-14T03:26:12.897 INFO:teuthology.task.install:extra packages: [] 2024-09-14T03:26:12.897 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:26:12.897 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:26:12.897 INFO:teuthology.packaging:ref: None 2024-09-14T03:26:12.898 INFO:teuthology.packaging:tag: None 2024-09-14T03:26:12.898 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:26:12.898 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:26:12.898 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-14T03:26:12.900 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:26:12.900 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:26:12.900 INFO:teuthology.packaging:ref: None 2024-09-14T03:26:12.900 INFO:teuthology.packaging:tag: None 2024-09-14T03:26:12.900 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:26:12.900 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:26:12.901 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-14T03:26:13.112 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/ 2024-09-14T03:26:13.113 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1629.geb16b97b 2024-09-14T03:26:13.129 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/ 2024-09-14T03:26:13.130 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1629.geb16b97b 2024-09-14T03:26:13.222 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:26:13.222 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:26:13.222 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:26:13.235 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:26:13.235 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:26:13.235 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:26:13.256 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-14T03:26:13.257 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:26:13.257 INFO:teuthology.packaging:ref: None 2024-09-14T03:26:13.257 INFO:teuthology.packaging:tag: None 2024-09-14T03:26:13.257 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:26:13.257 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:26:13.257 DEBUG:teuthology.orchestra.run.smithi195:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:26:13.263 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-14T03:26:13.263 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:26:13.263 INFO:teuthology.packaging:ref: None 2024-09-14T03:26:13.263 INFO:teuthology.packaging:tag: None 2024-09-14T03:26:13.263 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:26:13.263 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:26:13.263 DEBUG:teuthology.orchestra.run.smithi110:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:26:13.327 DEBUG:teuthology.orchestra.run.smithi195:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:26:13.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:26:13.408 DEBUG:teuthology.orchestra.run.smithi195:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:26:13.429 DEBUG:teuthology.orchestra.run.smithi110:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:26:13.452 INFO:teuthology.orchestra.run.smithi110.stdout:check_obsoletes = 1 2024-09-14T03:26:13.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum clean all 2024-09-14T03:26:13.474 INFO:teuthology.orchestra.run.smithi195.stdout:check_obsoletes = 1 2024-09-14T03:26:13.475 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum clean all 2024-09-14T03:26:13.677 INFO:teuthology.orchestra.run.smithi110.stdout:45 files removed 2024-09-14T03:26:13.703 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-14T03:26:13.709 INFO:teuthology.orchestra.run.smithi195.stdout:45 files removed 2024-09-14T03:26:13.736 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-14T03:26:14.261 INFO:teuthology.orchestra.run.smithi110.stdout:ceph packages for x86_64 282 kB/s | 77 kB 00:00 2024-09-14T03:26:14.294 INFO:teuthology.orchestra.run.smithi195.stdout:ceph packages for x86_64 282 kB/s | 77 kB 00:00 2024-09-14T03:26:14.528 INFO:teuthology.orchestra.run.smithi110.stdout:ceph noarch packages 48 kB/s | 11 kB 00:00 2024-09-14T03:26:14.578 INFO:teuthology.orchestra.run.smithi195.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-14T03:26:14.796 INFO:teuthology.orchestra.run.smithi110.stdout:ceph source packages 7.7 kB/s | 1.8 kB 00:00 2024-09-14T03:26:14.862 INFO:teuthology.orchestra.run.smithi195.stdout:ceph source packages 7.2 kB/s | 1.8 kB 00:00 2024-09-14T03:26:15.570 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.2 MB 00:00 2024-09-14T03:26:17.820 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - BaseOS 2.8 MB/s | 8.2 MB 00:02 2024-09-14T03:26:19.780 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - AppStream 23 MB/s | 20 MB 00:00 2024-09-14T03:26:25.523 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - CRB 3.4 MB/s | 6.5 MB 00:01 2024-09-14T03:26:28.040 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 19 kB 00:01 2024-09-14T03:26:29.370 INFO:teuthology.orchestra.run.smithi110.stdout:Extra Packages for Enterprise Linux 19 MB/s | 23 MB 00:01 2024-09-14T03:26:34.720 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - AppStream 1.1 MB/s | 20 MB 00:18 2024-09-14T03:26:34.842 INFO:teuthology.orchestra.run.smithi110.stdout:lab-extras 39 kB/s | 1.7 kB 00:00 2024-09-14T03:26:37.178 INFO:teuthology.orchestra.run.smithi110.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-14T03:26:37.179 INFO:teuthology.orchestra.run.smithi110.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-14T03:26:37.246 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2024-09-14T03:26:37.253 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================= 2024-09-14T03:26:37.253 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2024-09-14T03:26:37.254 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================= 2024-09-14T03:26:37.254 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2024-09-14T03:26:37.254 INFO:teuthology.orchestra.run.smithi110.stdout: ceph x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 6.5 k 2024-09-14T03:26:37.254 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-base x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 5.0 M 2024-09-14T03:26:37.254 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-fuse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 813 k 2024-09-14T03:26:37.255 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 137 k 2024-09-14T03:26:37.255 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 1.4 M 2024-09-14T03:26:37.255 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 124 k 2024-09-14T03:26:37.255 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 3.4 M 2024-09-14T03:26:37.256 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 7.4 M 2024-09-14T03:26:37.256 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-rook noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 47 k 2024-09-14T03:26:37.256 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-radosgw x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 14 M 2024-09-14T03:26:37.256 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-test x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 48 M 2024-09-14T03:26:37.256 INFO:teuthology.orchestra.run.smithi110.stdout: cephadm noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 83 k 2024-09-14T03:26:37.257 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs-devel x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 25 k 2024-09-14T03:26:37.257 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 636 k 2024-09-14T03:26:37.257 INFO:teuthology.orchestra.run.smithi110.stdout: librados-devel x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 126 k 2024-09-14T03:26:37.257 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cephfs x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 157 k 2024-09-14T03:26:37.257 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-14T03:26:37.258 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rados x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 320 k 2024-09-14T03:26:37.258 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rbd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 296 k 2024-09-14T03:26:37.258 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rgw x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 99 k 2024-09-14T03:26:37.258 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-fuse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 82 k 2024-09-14T03:26:37.258 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-mirror x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.9 M 2024-09-14T03:26:37.259 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-nbd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 165 k 2024-09-14T03:26:37.259 INFO:teuthology.orchestra.run.smithi110.stdout:Upgrading: 2024-09-14T03:26:37.259 INFO:teuthology.orchestra.run.smithi110.stdout: librados2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 3.1 M 2024-09-14T03:26:37.259 INFO:teuthology.orchestra.run.smithi110.stdout: librbd1 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.9 M 2024-09-14T03:26:37.259 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2024-09-14T03:26:37.260 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-common x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 23 M 2024-09-14T03:26:37.260 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 24 k 2024-09-14T03:26:37.260 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mds x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.1 M 2024-09-14T03:26:37.260 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 240 k 2024-09-14T03:26:37.260 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mon x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 3.9 M 2024-09-14T03:26:37.261 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-osd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 15 M 2024-09-14T03:26:37.261 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 15 k 2024-09-14T03:26:37.261 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-selinux x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 25 k 2024-09-14T03:26:37.261 INFO:teuthology.orchestra.run.smithi110.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T03:26:37.261 INFO:teuthology.orchestra.run.smithi110.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T03:26:37.262 INFO:teuthology.orchestra.run.smithi110.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T03:26:37.262 INFO:teuthology.orchestra.run.smithi110.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:26:37.262 INFO:teuthology.orchestra.run.smithi110.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:26:37.262 INFO:teuthology.orchestra.run.smithi110.stdout: libcephsqlite x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 163 k 2024-09-14T03:26:37.262 INFO:teuthology.orchestra.run.smithi110.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T03:26:37.262 INFO:teuthology.orchestra.run.smithi110.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:26:37.263 INFO:teuthology.orchestra.run.smithi110.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:26:37.263 INFO:teuthology.orchestra.run.smithi110.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T03:26:37.263 INFO:teuthology.orchestra.run.smithi110.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:26:37.263 INFO:teuthology.orchestra.run.smithi110.stdout: libradosstriper1 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 458 k 2024-09-14T03:26:37.263 INFO:teuthology.orchestra.run.smithi110.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:26:37.264 INFO:teuthology.orchestra.run.smithi110.stdout: librgw2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 4.9 M 2024-09-14T03:26:37.264 INFO:teuthology.orchestra.run.smithi110.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:26:37.264 INFO:teuthology.orchestra.run.smithi110.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:26:37.264 INFO:teuthology.orchestra.run.smithi110.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T03:26:37.264 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T03:26:37.264 INFO:teuthology.orchestra.run.smithi110.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T03:26:37.265 INFO:teuthology.orchestra.run.smithi110.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-14T03:26:37.265 INFO:teuthology.orchestra.run.smithi110.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T03:26:37.265 INFO:teuthology.orchestra.run.smithi110.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:26:37.265 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T03:26:37.265 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-argparse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 45 k 2024-09-14T03:26:37.266 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-common x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 120 k 2024-09-14T03:26:37.266 INFO:teuthology.orchestra.run.smithi110.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T03:26:37.266 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:26:37.266 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:26:37.266 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:26:37.267 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:26:37.267 INFO:teuthology.orchestra.run.smithi110.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T03:26:37.267 INFO:teuthology.orchestra.run.smithi110.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T03:26:37.267 INFO:teuthology.orchestra.run.smithi110.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-14T03:26:37.267 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:26:37.267 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:26:37.268 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:26:37.268 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:26:37.268 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:26:37.268 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T03:26:37.268 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T03:26:37.269 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T03:26:37.269 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-14T03:26:37.269 INFO:teuthology.orchestra.run.smithi110.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T03:26:37.269 INFO:teuthology.orchestra.run.smithi110.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:26:37.269 INFO:teuthology.orchestra.run.smithi110.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:26:37.270 INFO:teuthology.orchestra.run.smithi110.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:26:37.270 INFO:teuthology.orchestra.run.smithi110.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:26:37.270 INFO:teuthology.orchestra.run.smithi110.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T03:26:37.270 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T03:26:37.270 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T03:26:37.270 INFO:teuthology.orchestra.run.smithi110.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T03:26:37.271 INFO:teuthology.orchestra.run.smithi110.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-14T03:26:37.271 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:26:37.271 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-14T03:26:37.271 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:26:37.271 INFO:teuthology.orchestra.run.smithi110.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:26:37.272 INFO:teuthology.orchestra.run.smithi110.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:26:37.272 INFO:teuthology.orchestra.run.smithi110.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-14T03:26:37.272 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:26:37.272 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T03:26:37.272 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T03:26:37.272 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:26:37.273 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:26:37.273 INFO:teuthology.orchestra.run.smithi110.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T03:26:37.273 INFO:teuthology.orchestra.run.smithi110.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:26:37.273 INFO:teuthology.orchestra.run.smithi110.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T03:26:37.273 INFO:teuthology.orchestra.run.smithi110.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T03:26:37.274 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T03:26:37.274 INFO:teuthology.orchestra.run.smithi110.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T03:26:37.274 INFO:teuthology.orchestra.run.smithi110.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:26:37.274 INFO:teuthology.orchestra.run.smithi110.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:26:37.274 INFO:teuthology.orchestra.run.smithi110.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T03:26:37.274 INFO:teuthology.orchestra.run.smithi110.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:26:37.275 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:26:37.275 INFO:teuthology.orchestra.run.smithi110.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T03:26:37.275 INFO:teuthology.orchestra.run.smithi110.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:26:37.275 INFO:teuthology.orchestra.run.smithi110.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:26:37.275 INFO:teuthology.orchestra.run.smithi110.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:26:37.276 INFO:teuthology.orchestra.run.smithi110.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:26:37.276 INFO:teuthology.orchestra.run.smithi110.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:26:37.276 INFO:teuthology.orchestra.run.smithi110.stdout:Installing weak dependencies: 2024-09-14T03:26:37.276 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-14T03:26:37.276 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:37.276 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2024-09-14T03:26:37.277 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================= 2024-09-14T03:26:37.277 INFO:teuthology.orchestra.run.smithi110.stdout:Install 112 Packages 2024-09-14T03:26:37.277 INFO:teuthology.orchestra.run.smithi110.stdout:Upgrade 2 Packages 2024-09-14T03:26:37.277 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:26:37.278 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 190 M 2024-09-14T03:26:37.278 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2024-09-14T03:26:37.694 INFO:teuthology.orchestra.run.smithi110.stdout:(1/114): ceph-17.2.7-1629.geb16b97b.el9.x86_64. 52 kB/s | 6.5 kB 00:00 2024-09-14T03:26:37.877 INFO:teuthology.orchestra.run.smithi110.stdout:(2/114): ceph-fuse-17.2.7-1629.geb16b97b.el9.x8 4.3 MB/s | 813 kB 00:00 2024-09-14T03:26:37.977 INFO:teuthology.orchestra.run.smithi110.stdout:(3/114): ceph-immutable-object-cache-17.2.7-162 1.3 MB/s | 137 kB 00:00 2024-09-14T03:26:38.044 INFO:teuthology.orchestra.run.smithi110.stdout:(4/114): ceph-base-17.2.7-1629.geb16b97b.el9.x8 10 MB/s | 5.0 MB 00:00 2024-09-14T03:26:38.136 INFO:teuthology.orchestra.run.smithi110.stdout:(5/114): ceph-mgr-17.2.7-1629.geb16b97b.el9.x86 15 MB/s | 1.4 MB 00:00 2024-09-14T03:26:38.378 INFO:teuthology.orchestra.run.smithi110.stdout:(6/114): ceph-mon-17.2.7-1629.geb16b97b.el9.x86 16 MB/s | 3.9 MB 00:00 2024-09-14T03:26:38.520 INFO:teuthology.orchestra.run.smithi110.stdout:(7/114): ceph-mds-17.2.7-1629.geb16b97b.el9.x86 3.8 MB/s | 2.1 MB 00:00 2024-09-14T03:26:38.737 INFO:teuthology.orchestra.run.smithi110.stdout:(8/114): ceph-common-17.2.7-1629.geb16b97b.el9. 19 MB/s | 23 MB 00:01 2024-09-14T03:26:38.771 INFO:teuthology.orchestra.run.smithi110.stdout:(9/114): ceph-selinux-17.2.7-1629.geb16b97b.el9 759 kB/s | 25 kB 00:00 2024-09-14T03:26:39.065 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.5 MB 00:00 2024-09-14T03:26:39.213 INFO:teuthology.orchestra.run.smithi110.stdout:(10/114): ceph-osd-17.2.7-1629.geb16b97b.el9.x8 18 MB/s | 15 MB 00:00 2024-09-14T03:26:39.246 INFO:teuthology.orchestra.run.smithi110.stdout:(11/114): libcephfs-devel-17.2.7-1629.geb16b97b 751 kB/s | 25 kB 00:00 2024-09-14T03:26:39.330 INFO:teuthology.orchestra.run.smithi110.stdout:(12/114): libcephfs2-17.2.7-1629.geb16b97b.el9. 7.5 MB/s | 636 kB 00:00 2024-09-14T03:26:39.405 INFO:teuthology.orchestra.run.smithi110.stdout:(13/114): libcephsqlite-17.2.7-1629.geb16b97b.e 2.1 MB/s | 163 kB 00:00 2024-09-14T03:26:39.472 INFO:teuthology.orchestra.run.smithi110.stdout:(14/114): librados-devel-17.2.7-1629.geb16b97b. 1.9 MB/s | 126 kB 00:00 2024-09-14T03:26:39.539 INFO:teuthology.orchestra.run.smithi110.stdout:(15/114): libradosstriper1-17.2.7-1629.geb16b97 6.7 MB/s | 458 kB 00:00 2024-09-14T03:26:39.739 INFO:teuthology.orchestra.run.smithi110.stdout:(16/114): ceph-radosgw-17.2.7-1629.geb16b97b.el 12 MB/s | 14 MB 00:01 2024-09-14T03:26:39.782 INFO:teuthology.orchestra.run.smithi110.stdout:(17/114): python3-ceph-argparse-17.2.7-1629.geb 1.1 MB/s | 45 kB 00:00 2024-09-14T03:26:39.865 INFO:teuthology.orchestra.run.smithi110.stdout:(18/114): librgw2-17.2.7-1629.geb16b97b.el9.x86 15 MB/s | 4.9 MB 00:00 2024-09-14T03:26:39.890 INFO:teuthology.orchestra.run.smithi110.stdout:(19/114): python3-ceph-common-17.2.7-1629.geb16 1.1 MB/s | 120 kB 00:00 2024-09-14T03:26:39.916 INFO:teuthology.orchestra.run.smithi110.stdout:(20/114): python3-cephfs-17.2.7-1629.geb16b97b. 3.0 MB/s | 157 kB 00:00 2024-09-14T03:26:39.941 INFO:teuthology.orchestra.run.smithi110.stdout:(21/114): python3-rados-17.2.7-1629.geb16b97b.e 6.2 MB/s | 320 kB 00:00 2024-09-14T03:26:39.966 INFO:teuthology.orchestra.run.smithi110.stdout:(22/114): python3-rbd-17.2.7-1629.geb16b97b.el9 5.8 MB/s | 296 kB 00:00 2024-09-14T03:26:39.992 INFO:teuthology.orchestra.run.smithi110.stdout:(23/114): python3-rgw-17.2.7-1629.geb16b97b.el9 1.9 MB/s | 99 kB 00:00 2024-09-14T03:26:40.017 INFO:teuthology.orchestra.run.smithi110.stdout:(24/114): rbd-fuse-17.2.7-1629.geb16b97b.el9.x8 1.6 MB/s | 82 kB 00:00 2024-09-14T03:26:40.075 INFO:teuthology.orchestra.run.smithi110.stdout:(25/114): rbd-nbd-17.2.7-1629.geb16b97b.el9.x86 2.8 MB/s | 165 kB 00:00 2024-09-14T03:26:40.142 INFO:teuthology.orchestra.run.smithi110.stdout:(26/114): ceph-grafana-dashboards-17.2.7-1629.g 366 kB/s | 24 kB 00:00 2024-09-14T03:26:40.201 INFO:teuthology.orchestra.run.smithi110.stdout:(27/114): rbd-mirror-17.2.7-1629.geb16b97b.el9. 14 MB/s | 2.9 MB 00:00 2024-09-14T03:26:40.229 INFO:teuthology.orchestra.run.smithi110.stdout:(28/114): ceph-mgr-cephadm-17.2.7-1629.geb16b97 1.4 MB/s | 124 kB 00:00 2024-09-14T03:26:40.519 INFO:teuthology.orchestra.run.smithi110.stdout:(29/114): ceph-mgr-dashboard-17.2.7-1629.geb16b 11 MB/s | 3.4 MB 00:00 2024-09-14T03:26:40.555 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - Extras packages 70 kB/s | 19 kB 00:00 2024-09-14T03:26:40.660 INFO:teuthology.orchestra.run.smithi110.stdout:(30/114): ceph-mgr-diskprediction-local-17.2.7- 17 MB/s | 7.4 MB 00:00 2024-09-14T03:26:40.685 INFO:teuthology.orchestra.run.smithi110.stdout:(31/114): ceph-mgr-modules-core-17.2.7-1629.geb 1.4 MB/s | 240 kB 00:00 2024-09-14T03:26:40.702 INFO:teuthology.orchestra.run.smithi110.stdout:(32/114): ceph-mgr-rook-17.2.7-1629.geb16b97b.e 1.1 MB/s | 47 kB 00:00 2024-09-14T03:26:41.128 INFO:teuthology.orchestra.run.smithi110.stdout:(33/114): ceph-test-17.2.7-1629.geb16b97b.el9.x 20 MB/s | 48 MB 00:02 2024-09-14T03:26:41.145 INFO:teuthology.orchestra.run.smithi110.stdout:(34/114): ceph-prometheus-alerts-17.2.7-1629.ge 32 kB/s | 15 kB 00:00 2024-09-14T03:26:41.192 INFO:teuthology.orchestra.run.smithi110.stdout:(35/114): cephadm-17.2.7-1629.geb16b97b.el9.noa 169 kB/s | 83 kB 00:00 2024-09-14T03:26:41.262 INFO:teuthology.orchestra.run.smithi110.stdout:(36/114): fuse-2.9.9-16.el9.x86_64.rpm 598 kB/s | 80 kB 00:00 2024-09-14T03:26:41.321 INFO:teuthology.orchestra.run.smithi110.stdout:(37/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.2 MB/s | 253 kB 00:00 2024-09-14T03:26:41.346 INFO:teuthology.orchestra.run.smithi110.stdout:(38/114): libquadmath-11.5.0-2.el9.x86_64.rpm 1.2 MB/s | 191 kB 00:00 2024-09-14T03:26:41.371 INFO:teuthology.orchestra.run.smithi110.stdout:(39/114): libgfortran-11.5.0-2.el9.x86_64.rpm 3.5 MB/s | 801 kB 00:00 2024-09-14T03:26:41.388 INFO:teuthology.orchestra.run.smithi110.stdout:(40/114): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-09-14T03:26:41.401 INFO:teuthology.orchestra.run.smithi195.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-14T03:26:41.413 INFO:teuthology.orchestra.run.smithi110.stdout:(41/114): python3-pycparser-2.20-6.el9.noarch.r 3.1 MB/s | 135 kB 00:00 2024-09-14T03:26:41.439 INFO:teuthology.orchestra.run.smithi110.stdout:(42/114): python3-requests-2.25.1-8.el9.noarch. 2.4 MB/s | 125 kB 00:00 2024-09-14T03:26:41.464 INFO:teuthology.orchestra.run.smithi110.stdout:(43/114): python3-urllib3-1.26.5-6.el9.noarch.r 4.2 MB/s | 215 kB 00:00 2024-09-14T03:26:41.498 INFO:teuthology.orchestra.run.smithi110.stdout:(44/114): python3-cryptography-36.0.1-4.el9.x86 6.9 MB/s | 1.2 MB 00:00 2024-09-14T03:26:41.573 INFO:teuthology.orchestra.run.smithi110.stdout:(45/114): flexiblas-3.0.4-8.el9.x86_64.rpm 244 kB/s | 33 kB 00:00 2024-09-14T03:26:41.598 INFO:teuthology.orchestra.run.smithi110.stdout:(46/114): flexiblas-openblas-openmp-3.0.4-8.el9 182 kB/s | 18 kB 00:00 2024-09-14T03:26:41.682 INFO:teuthology.orchestra.run.smithi110.stdout:(47/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 542 kB/s | 45 kB 00:00 2024-09-14T03:26:41.707 INFO:teuthology.orchestra.run.smithi110.stdout:(48/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.2 MB/s | 160 kB 00:00 2024-09-14T03:26:41.841 INFO:teuthology.orchestra.run.smithi110.stdout:(49/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.1 MB/s | 292 kB 00:00 2024-09-14T03:26:41.891 INFO:teuthology.orchestra.run.smithi110.stdout:(50/114): openblas-0.3.26-2.el9.x86_64.rpm 770 kB/s | 38 kB 00:00 2024-09-14T03:26:41.958 INFO:teuthology.orchestra.run.smithi110.stdout:(51/114): librdkafka-1.6.1-102.el9.x86_64.rpm 2.3 MB/s | 662 kB 00:00 2024-09-14T03:26:42.009 INFO:teuthology.orchestra.run.smithi110.stdout:(52/114): python3-attrs-20.3.0-7.el9.noarch.rpm 1.7 MB/s | 87 kB 00:00 2024-09-14T03:26:42.475 INFO:teuthology.orchestra.run.smithi110.stdout:(53/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 3.0 MB/s | 3.0 MB 00:01 2024-09-14T03:26:42.559 INFO:teuthology.orchestra.run.smithi110.stdout:(54/114): python3-devel-3.9.19-8.el9.x86_64.rpm 2.9 MB/s | 245 kB 00:00 2024-09-14T03:26:42.609 INFO:teuthology.orchestra.run.smithi110.stdout:(55/114): python3-iniconfig-1.1.1-7.el9.noarch. 349 kB/s | 17 kB 00:00 2024-09-14T03:26:42.709 INFO:teuthology.orchestra.run.smithi110.stdout:(56/114): python3-jinja2-2.11.3-6.el9.noarch.rp 2.4 MB/s | 248 kB 00:00 2024-09-14T03:26:42.760 INFO:teuthology.orchestra.run.smithi110.stdout:(57/114): python3-jsonpatch-1.21-16.el9.noarch. 513 kB/s | 26 kB 00:00 2024-09-14T03:26:42.810 INFO:teuthology.orchestra.run.smithi110.stdout:(58/114): python3-jsonpointer-2.0-4.el9.noarch. 382 kB/s | 19 kB 00:00 2024-09-14T03:26:42.914 INFO:teuthology.orchestra.run.smithi110.stdout:(59/114): python3-mako-1.1.4-6.el9.noarch.rpm 1.6 MB/s | 172 kB 00:00 2024-09-14T03:26:42.989 INFO:teuthology.orchestra.run.smithi110.stdout:(60/114): python3-markupsafe-1.1.1-12.el9.x86_6 464 kB/s | 35 kB 00:00 2024-09-14T03:26:43.415 INFO:teuthology.orchestra.run.smithi110.stdout:(61/114): openblas-openmp-0.3.26-2.el9.x86_64.r 3.2 MB/s | 4.9 MB 00:01 2024-09-14T03:26:43.523 INFO:teuthology.orchestra.run.smithi110.stdout:(62/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 2.3 MB/s | 250 kB 00:00 2024-09-14T03:26:43.607 INFO:teuthology.orchestra.run.smithi110.stdout:(63/114): python3-oauthlib-3.1.1-5.el9.noarch.r 2.6 MB/s | 222 kB 00:00 2024-09-14T03:26:43.670 INFO:teuthology.orchestra.run.smithi110.stdout:(64/114): python3-packaging-20.9-5.el9.noarch.r 1.3 MB/s | 77 kB 00:00 2024-09-14T03:26:43.732 INFO:teuthology.orchestra.run.smithi110.stdout:(65/114): python3-babel-2.9.1-2.el9.noarch.rpm 3.5 MB/s | 6.0 MB 00:01 2024-09-14T03:26:43.758 INFO:teuthology.orchestra.run.smithi110.stdout:(66/114): python3-pluggy-0.13.1-7.el9.noarch.rp 450 kB/s | 41 kB 00:00 2024-09-14T03:26:43.783 INFO:teuthology.orchestra.run.smithi110.stdout:(67/114): python3-prettytable-0.7.2-27.el9.noar 842 kB/s | 42 kB 00:00 2024-09-14T03:26:43.834 INFO:teuthology.orchestra.run.smithi110.stdout:(68/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 3.1 MB/s | 159 kB 00:00 2024-09-14T03:26:43.884 INFO:teuthology.orchestra.run.smithi110.stdout:(69/114): python3-py-1.10.0-6.el9.noarch.rpm 3.7 MB/s | 477 kB 00:00 2024-09-14T03:26:43.934 INFO:teuthology.orchestra.run.smithi110.stdout:(70/114): python3-pyasn1-modules-0.4.8-6.el9.no 2.7 MB/s | 279 kB 00:00 2024-09-14T03:26:43.993 INFO:teuthology.orchestra.run.smithi110.stdout:(71/114): python3-pytz-2021.1-5.el9.noarch.rpm 879 kB/s | 51 kB 00:00 2024-09-14T03:26:44.026 INFO:teuthology.orchestra.run.smithi110.stdout:(72/114): python3-pytest-6.2.2-7.el9.noarch.rpm 3.6 MB/s | 519 kB 00:00 2024-09-14T03:26:44.052 INFO:teuthology.orchestra.run.smithi110.stdout:(73/114): python3-requests-oauthlib-1.3.0-12.el 915 kB/s | 54 kB 00:00 2024-09-14T03:26:44.102 INFO:teuthology.orchestra.run.smithi110.stdout:(74/114): python3-toml-0.10.2-6.el9.noarch.rpm 834 kB/s | 42 kB 00:00 2024-09-14T03:26:44.227 INFO:teuthology.orchestra.run.smithi110.stdout:(75/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.1 MB/s | 5.1 MB 00:01 2024-09-14T03:26:44.253 INFO:teuthology.orchestra.run.smithi110.stdout:(76/114): socat-1.7.4.1-6.el9.x86_64.rpm 2.0 MB/s | 304 kB 00:00 2024-09-14T03:26:44.286 INFO:teuthology.orchestra.run.smithi110.stdout:(77/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.1 MB/s | 64 kB 00:00 2024-09-14T03:26:44.328 INFO:teuthology.orchestra.run.smithi110.stdout:(78/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 4.0 MB/s | 308 kB 00:00 2024-09-14T03:26:44.353 INFO:teuthology.orchestra.run.smithi110.stdout:(79/114): libunwind-1.6.2-1.el9.x86_64.rpm 2.9 MB/s | 67 kB 00:00 2024-09-14T03:26:44.379 INFO:teuthology.orchestra.run.smithi110.stdout:(80/114): liboath-2.6.7-2.el9.x86_64.rpm 537 kB/s | 49 kB 00:00 2024-09-14T03:26:44.404 INFO:teuthology.orchestra.run.smithi110.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.7 MB/s | 43 kB 00:00 2024-09-14T03:26:44.429 INFO:teuthology.orchestra.run.smithi110.stdout:(82/114): python3-cachetools-4.2.4-1.el9.noarch 1.3 MB/s | 32 kB 00:00 2024-09-14T03:26:44.455 INFO:teuthology.orchestra.run.smithi110.stdout:(83/114): python3-certifi-2023.05.07-4.el9.noar 564 kB/s | 14 kB 00:00 2024-09-14T03:26:44.480 INFO:teuthology.orchestra.run.smithi110.stdout:(84/114): python3-asyncssh-2.13.2-5.el9.noarch. 4.2 MB/s | 548 kB 00:00 2024-09-14T03:26:44.505 INFO:teuthology.orchestra.run.smithi110.stdout:(85/114): python3-cheroot-8.6.0-4.el9.noarch.rp 3.3 MB/s | 172 kB 00:00 2024-09-14T03:26:44.539 INFO:teuthology.orchestra.run.smithi110.stdout:(86/114): python3-cherrypy-18.6.1-2.el9.noarch. 6.0 MB/s | 358 kB 00:00 2024-09-14T03:26:44.564 INFO:teuthology.orchestra.run.smithi110.stdout:(87/114): python3-google-auth-2.34.0-1.el9.noar 3.7 MB/s | 220 kB 00:00 2024-09-14T03:26:44.589 INFO:teuthology.orchestra.run.smithi110.stdout:(88/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 430 kB/s | 11 kB 00:00 2024-09-14T03:26:44.614 INFO:teuthology.orchestra.run.smithi110.stdout:(89/114): python3-jaraco-classes-3.2.1-5.el9.no 353 kB/s | 18 kB 00:00 2024-09-14T03:26:44.640 INFO:teuthology.orchestra.run.smithi110.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 461 kB/s | 23 kB 00:00 2024-09-14T03:26:44.665 INFO:teuthology.orchestra.run.smithi110.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 386 kB/s | 19 kB 00:00 2024-09-14T03:26:44.692 INFO:teuthology.orchestra.run.smithi110.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 393 kB/s | 20 kB 00:00 2024-09-14T03:26:44.716 INFO:teuthology.orchestra.run.smithi110.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 181 kB/s | 9.0 kB 00:00 2024-09-14T03:26:44.741 INFO:teuthology.orchestra.run.smithi110.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 812 kB/s | 41 kB 00:00 2024-09-14T03:26:44.766 INFO:teuthology.orchestra.run.smithi110.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 1.8 MB/s | 46 kB 00:00 2024-09-14T03:26:44.783 INFO:teuthology.orchestra.run.smithi110.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 4.6 MB/s | 79 kB 00:00 2024-09-14T03:26:44.817 INFO:teuthology.orchestra.run.smithi110.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 1.7 MB/s | 58 kB 00:00 2024-09-14T03:26:44.851 INFO:teuthology.orchestra.run.smithi110.stdout:(98/114): python3-pecan-1.4.2-3.el9.noarch.rpm 8.1 MB/s | 272 kB 00:00 2024-09-14T03:26:44.876 INFO:teuthology.orchestra.run.smithi110.stdout:(99/114): python3-portend-3.1.0-2.el9.noarch.rp 654 kB/s | 16 kB 00:00 2024-09-14T03:26:44.901 INFO:teuthology.orchestra.run.smithi110.stdout:(100/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 3.6 MB/s | 90 kB 00:00 2024-09-14T03:26:44.927 INFO:teuthology.orchestra.run.smithi110.stdout:(101/114): python3-repoze-lru-0.7-16.el9.noarch 1.2 MB/s | 31 kB 00:00 2024-09-14T03:26:44.977 INFO:teuthology.orchestra.run.smithi110.stdout:(102/114): python3-kubernetes-26.1.0-2.el9.noar 3.9 MB/s | 1.0 MB 00:00 2024-09-14T03:26:45.002 INFO:teuthology.orchestra.run.smithi110.stdout:(103/114): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-09-14T03:26:45.027 INFO:teuthology.orchestra.run.smithi110.stdout:(104/114): python3-tempora-5.0.0-2.el9.noarch.r 1.4 MB/s | 36 kB 00:00 2024-09-14T03:26:45.053 INFO:teuthology.orchestra.run.smithi110.stdout:(105/114): python3-routes-2.5.1-5.el9.noarch.rp 1.5 MB/s | 188 kB 00:00 2024-09-14T03:26:45.078 INFO:teuthology.orchestra.run.smithi110.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 1.5 MB/s | 74 kB 00:00 2024-09-14T03:26:45.104 INFO:teuthology.orchestra.run.smithi110.stdout:(107/114): python3-websocket-client-1.2.3-2.el9 3.5 MB/s | 90 kB 00:00 2024-09-14T03:26:45.129 INFO:teuthology.orchestra.run.smithi110.stdout:(108/114): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-14T03:26:45.146 INFO:teuthology.orchestra.run.smithi110.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 1.2 MB/s | 20 kB 00:00 2024-09-14T03:26:45.179 INFO:teuthology.orchestra.run.smithi110.stdout:(110/114): python3-werkzeug-2.0.3-3.el9.1.noarc 5.5 MB/s | 427 kB 00:00 2024-09-14T03:26:45.230 INFO:teuthology.orchestra.run.smithi110.stdout:(111/114): thrift-0.15.0-2.el9.x86_64.rpm 19 MB/s | 1.6 MB 00:00 2024-09-14T03:26:45.438 INFO:teuthology.orchestra.run.smithi110.stdout:(112/114): librados2-17.2.7-1629.geb16b97b.el9. 12 MB/s | 3.1 MB 00:00 2024-09-14T03:26:45.497 INFO:teuthology.orchestra.run.smithi110.stdout:(113/114): librbd1-17.2.7-1629.geb16b97b.el9.x8 11 MB/s | 2.9 MB 00:00 2024-09-14T03:26:45.730 INFO:teuthology.orchestra.run.smithi110.stdout:(114/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 9.5 MB/s | 16 MB 00:01 2024-09-14T03:26:45.735 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:26:45.735 INFO:teuthology.orchestra.run.smithi110.stdout:Total 22 MB/s | 190 MB 00:08 2024-09-14T03:26:46.488 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2024-09-14T03:26:46.582 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2024-09-14T03:26:46.582 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2024-09-14T03:26:46.798 INFO:teuthology.orchestra.run.smithi195.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-14T03:26:48.247 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2024-09-14T03:26:48.248 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2024-09-14T03:26:49.141 INFO:teuthology.orchestra.run.smithi195.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-14T03:26:49.142 INFO:teuthology.orchestra.run.smithi195.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-14T03:26:49.211 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout:======================================================================================= 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout:======================================================================================= 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout: ceph x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 6.5 k 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-base x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 5.0 M 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-fuse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 813 k 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 137 k 2024-09-14T03:26:49.218 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 1.4 M 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 124 k 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 3.4 M 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 7.4 M 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-rook noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 47 k 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-radosgw x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 14 M 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-test x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 48 M 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: cephadm noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 83 k 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs-devel x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 25 k 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 636 k 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: librados-devel x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 126 k 2024-09-14T03:26:49.219 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cephfs x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 157 k 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rados x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 320 k 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rbd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 296 k 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rgw x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 99 k 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-fuse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 82 k 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-mirror x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.9 M 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-nbd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 165 k 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout:Upgrading: 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: librados2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 3.1 M 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout: librbd1 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.9 M 2024-09-14T03:26:49.220 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-common x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 23 M 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 24 k 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mds x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.1 M 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 240 k 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mon x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 3.9 M 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-osd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 15 M 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 15 k 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-selinux x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 25 k 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T03:26:49.221 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: libcephsqlite x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 163 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: libradosstriper1 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 458 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: librgw2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 4.9 M 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:26:49.222 INFO:teuthology.orchestra.run.smithi195.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-argparse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 45 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-common x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 120 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:26:49.223 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T03:26:49.224 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T03:26:49.225 INFO:teuthology.orchestra.run.smithi195.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T03:26:49.226 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:26:49.227 INFO:teuthology.orchestra.run.smithi195.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout:Installing weak dependencies: 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout:======================================================================================= 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout:Install 112 Packages 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout:Upgrade 2 Packages 2024-09-14T03:26:49.228 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:26:49.229 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 190 M 2024-09-14T03:26:49.229 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-09-14T03:26:50.275 INFO:teuthology.orchestra.run.smithi195.stdout:(1/114): ceph-17.2.7-1629.geb16b97b.el9.x86_64. 45 kB/s | 6.5 kB 00:00 2024-09-14T03:26:50.583 INFO:teuthology.orchestra.run.smithi195.stdout:(2/114): ceph-base-17.2.7-1629.geb16b97b.el9.x8 11 MB/s | 5.0 MB 00:00 2024-09-14T03:26:50.621 INFO:teuthology.orchestra.run.smithi195.stdout:(3/114): ceph-fuse-17.2.7-1629.geb16b97b.el9.x8 2.3 MB/s | 813 kB 00:00 2024-09-14T03:26:50.643 INFO:teuthology.orchestra.run.smithi195.stdout:(4/114): ceph-immutable-object-cache-17.2.7-162 2.3 MB/s | 137 kB 00:00 2024-09-14T03:26:50.726 INFO:teuthology.orchestra.run.smithi195.stdout:(5/114): ceph-mgr-17.2.7-1629.geb16b97b.el9.x86 17 MB/s | 1.4 MB 00:00 2024-09-14T03:26:50.968 INFO:teuthology.orchestra.run.smithi195.stdout:(6/114): ceph-mon-17.2.7-1629.geb16b97b.el9.x86 16 MB/s | 3.9 MB 00:00 2024-09-14T03:26:51.031 INFO:teuthology.orchestra.run.smithi195.stdout:(7/114): ceph-mds-17.2.7-1629.geb16b97b.el9.x86 5.1 MB/s | 2.1 MB 00:00 2024-09-14T03:26:51.294 INFO:teuthology.orchestra.run.smithi195.stdout:(8/114): ceph-common-17.2.7-1629.geb16b97b.el9. 19 MB/s | 23 MB 00:01 2024-09-14T03:26:51.335 INFO:teuthology.orchestra.run.smithi195.stdout:(9/114): ceph-selinux-17.2.7-1629.geb16b97b.el9 607 kB/s | 25 kB 00:00 2024-09-14T03:26:52.019 INFO:teuthology.orchestra.run.smithi195.stdout:(10/114): ceph-osd-17.2.7-1629.geb16b97b.el9.x8 15 MB/s | 15 MB 00:01 2024-09-14T03:26:52.061 INFO:teuthology.orchestra.run.smithi195.stdout:(11/114): libcephfs-devel-17.2.7-1629.geb16b97b 601 kB/s | 25 kB 00:00 2024-09-14T03:26:52.153 INFO:teuthology.orchestra.run.smithi195.stdout:(12/114): libcephfs2-17.2.7-1629.geb16b97b.el9. 6.8 MB/s | 636 kB 00:00 2024-09-14T03:26:52.211 INFO:teuthology.orchestra.run.smithi195.stdout:(13/114): libcephsqlite-17.2.7-1629.geb16b97b.e 2.7 MB/s | 163 kB 00:00 2024-09-14T03:26:52.278 INFO:teuthology.orchestra.run.smithi195.stdout:(14/114): librados-devel-17.2.7-1629.geb16b97b. 1.9 MB/s | 126 kB 00:00 2024-09-14T03:26:52.362 INFO:teuthology.orchestra.run.smithi195.stdout:(15/114): libradosstriper1-17.2.7-1629.geb16b97 5.4 MB/s | 458 kB 00:00 2024-09-14T03:26:52.950 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2024-09-14T03:26:53.008 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-14T03:26:53.042 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-14T03:26:53.054 INFO:teuthology.orchestra.run.smithi195.stdout:(16/114): ceph-test-17.2.7-1629.geb16b97b.el9.x 28 MB/s | 48 MB 00:01 2024-09-14T03:26:53.103 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-14T03:26:53.138 INFO:teuthology.orchestra.run.smithi195.stdout:(17/114): librgw2-17.2.7-1629.geb16b97b.el9.x86 6.3 MB/s | 4.9 MB 00:00 2024-09-14T03:26:53.154 INFO:teuthology.orchestra.run.smithi195.stdout:(18/114): python3-ceph-argparse-17.2.7-1629.geb 451 kB/s | 45 kB 00:00 2024-09-14T03:26:53.197 INFO:teuthology.orchestra.run.smithi195.stdout:(19/114): python3-ceph-common-17.2.7-1629.geb16 2.0 MB/s | 120 kB 00:00 2024-09-14T03:26:53.232 INFO:teuthology.orchestra.run.smithi195.stdout:(20/114): python3-cephfs-17.2.7-1629.geb16b97b. 2.0 MB/s | 157 kB 00:00 2024-09-14T03:26:53.282 INFO:teuthology.orchestra.run.smithi195.stdout:(21/114): python3-rados-17.2.7-1629.geb16b97b.e 3.7 MB/s | 320 kB 00:00 2024-09-14T03:26:53.308 INFO:teuthology.orchestra.run.smithi195.stdout:(22/114): python3-rbd-17.2.7-1629.geb16b97b.el9 3.8 MB/s | 296 kB 00:00 2024-09-14T03:26:53.333 INFO:teuthology.orchestra.run.smithi195.stdout:(23/114): python3-rgw-17.2.7-1629.geb16b97b.el9 1.9 MB/s | 99 kB 00:00 2024-09-14T03:26:53.358 INFO:teuthology.orchestra.run.smithi195.stdout:(24/114): rbd-fuse-17.2.7-1629.geb16b97b.el9.x8 1.6 MB/s | 82 kB 00:00 2024-09-14T03:26:53.411 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-14T03:26:53.416 INFO:teuthology.orchestra.run.smithi195.stdout:(25/114): rbd-nbd-17.2.7-1629.geb16b97b.el9.x86 2.8 MB/s | 165 kB 00:00 2024-09-14T03:26:53.467 INFO:teuthology.orchestra.run.smithi195.stdout:(26/114): ceph-grafana-dashboards-17.2.7-1629.g 489 kB/s | 24 kB 00:00 2024-09-14T03:26:53.517 INFO:teuthology.orchestra.run.smithi195.stdout:(27/114): rbd-mirror-17.2.7-1629.geb16b97b.el9. 16 MB/s | 2.9 MB 00:00 2024-09-14T03:26:53.526 INFO:teuthology.orchestra.run.smithi110.stdout: Upgrading : librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 5/116 2024-09-14T03:26:53.542 INFO:teuthology.orchestra.run.smithi195.stdout:(28/114): ceph-mgr-cephadm-17.2.7-1629.geb16b97 1.6 MB/s | 124 kB 00:00 2024-09-14T03:26:53.640 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 5/116 2024-09-14T03:26:53.669 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:26:53.718 INFO:teuthology.orchestra.run.smithi195.stdout:(29/114): ceph-radosgw-17.2.7-1629.geb16b97b.el 5.4 MB/s | 14 MB 00:02 2024-09-14T03:26:53.752 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:26:53.802 INFO:teuthology.orchestra.run.smithi195.stdout:(30/114): ceph-mgr-dashboard-17.2.7-1629.geb16b 12 MB/s | 3.4 MB 00:00 2024-09-14T03:26:53.810 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 7/116 2024-09-14T03:26:53.827 INFO:teuthology.orchestra.run.smithi195.stdout:(31/114): ceph-mgr-modules-core-17.2.7-1629.geb 2.2 MB/s | 240 kB 00:00 2024-09-14T03:26:53.844 INFO:teuthology.orchestra.run.smithi195.stdout:(32/114): ceph-mgr-rook-17.2.7-1629.geb16b97b.e 1.1 MB/s | 47 kB 00:00 2024-09-14T03:26:53.861 INFO:teuthology.orchestra.run.smithi195.stdout:(33/114): ceph-prometheus-alerts-17.2.7-1629.ge 438 kB/s | 15 kB 00:00 2024-09-14T03:26:53.881 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-14T03:26:53.903 INFO:teuthology.orchestra.run.smithi195.stdout:(34/114): cephadm-17.2.7-1629.geb16b97b.el9.noa 1.4 MB/s | 83 kB 00:00 2024-09-14T03:26:54.003 INFO:teuthology.orchestra.run.smithi195.stdout:(35/114): ceph-mgr-diskprediction-local-17.2.7- 16 MB/s | 7.4 MB 00:00 2024-09-14T03:26:54.513 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-14T03:26:54.539 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:26:54.570 INFO:teuthology.orchestra.run.smithi195.stdout:(36/114): fuse-2.9.9-16.el9.x86_64.rpm 113 kB/s | 80 kB 00:00 2024-09-14T03:26:54.613 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:26:54.640 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:26:54.740 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:26:54.753 INFO:teuthology.orchestra.run.smithi195.stdout:(37/114): libquadmath-11.5.0-2.el9.x86_64.rpm 255 kB/s | 191 kB 00:00 2024-09-14T03:26:54.766 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 12/116 2024-09-14T03:26:54.829 INFO:teuthology.orchestra.run.smithi195.stdout:(38/114): libgfortran-11.5.0-2.el9.x86_64.rpm 866 kB/s | 801 kB 00:00 2024-09-14T03:26:54.902 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 12/116 2024-09-14T03:26:54.937 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-14T03:26:54.969 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-14T03:26:55.027 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-14T03:26:55.065 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-14T03:26:55.139 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-14T03:26:55.183 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-14T03:26:55.223 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-14T03:26:55.250 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-14T03:26:55.321 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-14T03:26:55.373 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-14T03:26:55.412 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-14T03:26:55.437 INFO:teuthology.orchestra.run.smithi195.stdout:(39/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 292 kB/s | 253 kB 00:00 2024-09-14T03:26:55.471 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-14T03:26:55.496 INFO:teuthology.orchestra.run.smithi195.stdout:(40/114): python3-ply-3.11-14.el9.noarch.rpm 160 kB/s | 106 kB 00:00 2024-09-14T03:26:55.499 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-14T03:26:55.543 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el 26/116 2024-09-14T03:26:55.616 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_6 27/116 2024-09-14T03:26:55.746 INFO:teuthology.orchestra.run.smithi195.stdout:(41/114): python3-cryptography-36.0.1-4.el9.x86 1.2 MB/s | 1.2 MB 00:00 2024-09-14T03:26:55.767 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-14T03:26:55.828 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-14T03:26:56.146 INFO:teuthology.orchestra.run.smithi195.stdout:(42/114): python3-pycparser-2.20-6.el9.noarch.r 191 kB/s | 135 kB 00:00 2024-09-14T03:26:56.214 INFO:teuthology.orchestra.run.smithi195.stdout:(43/114): flexiblas-3.0.4-8.el9.x86_64.rpm 494 kB/s | 33 kB 00:00 2024-09-14T03:26:56.238 INFO:teuthology.orchestra.run.smithi195.stdout:(44/114): python3-requests-2.25.1-8.el9.noarch. 168 kB/s | 125 kB 00:00 2024-09-14T03:26:56.297 INFO:teuthology.orchestra.run.smithi195.stdout:(45/114): flexiblas-openblas-openmp-3.0.4-8.el9 312 kB/s | 18 kB 00:00 2024-09-14T03:26:56.360 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-14T03:26:56.364 INFO:teuthology.orchestra.run.smithi195.stdout:(46/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 2.3 MB/s | 160 kB 00:00 2024-09-14T03:26:56.406 INFO:teuthology.orchestra.run.smithi195.stdout:(47/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.1 MB/s | 45 kB 00:00 2024-09-14T03:26:56.449 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-14T03:26:56.486 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-14T03:26:56.521 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-14T03:26:56.523 INFO:teuthology.orchestra.run.smithi195.stdout:(48/114): librdkafka-1.6.1-102.el9.x86_64.rpm 5.5 MB/s | 662 kB 00:00 2024-09-14T03:26:56.548 INFO:teuthology.orchestra.run.smithi195.stdout:(49/114): python3-urllib3-1.26.5-6.el9.noarch.r 271 kB/s | 215 kB 00:00 2024-09-14T03:26:56.555 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-14T03:26:56.594 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-14T03:26:56.615 INFO:teuthology.orchestra.run.smithi195.stdout:(50/114): openblas-0.3.26-2.el9.x86_64.rpm 580 kB/s | 38 kB 00:00 2024-09-14T03:26:56.631 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 36/116 2024-09-14T03:26:56.641 INFO:teuthology.orchestra.run.smithi195.stdout:(51/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 292 kB 00:00 2024-09-14T03:26:56.677 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_6 37/116 2024-09-14T03:26:56.699 INFO:teuthology.orchestra.run.smithi195.stdout:(52/114): python3-attrs-20.3.0-7.el9.noarch.rpm 1.5 MB/s | 87 kB 00:00 2024-09-14T03:26:56.707 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-14T03:26:56.744 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-14T03:26:56.800 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-14T03:26:56.873 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-14T03:26:56.904 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-14T03:26:56.939 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-14T03:26:56.950 INFO:teuthology.orchestra.run.smithi195.stdout:(53/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 4.1 MB/s | 3.0 MB 00:00 2024-09-14T03:26:56.986 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-14T03:26:57.032 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-14T03:26:57.033 INFO:teuthology.orchestra.run.smithi195.stdout:(54/114): python3-devel-3.9.19-8.el9.x86_64.rpm 2.9 MB/s | 245 kB 00:00 2024-09-14T03:26:57.074 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-14T03:26:57.075 INFO:teuthology.orchestra.run.smithi195.stdout:(55/114): python3-iniconfig-1.1.1-7.el9.noarch. 421 kB/s | 17 kB 00:00 2024-09-14T03:26:57.200 INFO:teuthology.orchestra.run.smithi195.stdout:(56/114): python3-jinja2-2.11.3-6.el9.noarch.rp 1.9 MB/s | 248 kB 00:00 2024-09-14T03:26:57.212 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-14T03:26:57.242 INFO:teuthology.orchestra.run.smithi195.stdout:(57/114): python3-jsonpatch-1.21-16.el9.noarch. 615 kB/s | 26 kB 00:00 2024-09-14T03:26:57.250 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-14T03:26:57.284 INFO:teuthology.orchestra.run.smithi195.stdout:(58/114): python3-jsonpointer-2.0-4.el9.noarch. 461 kB/s | 19 kB 00:00 2024-09-14T03:26:57.292 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-14T03:26:57.360 INFO:teuthology.orchestra.run.smithi195.stdout:(59/114): python3-mako-1.1.4-6.el9.noarch.rpm 2.2 MB/s | 172 kB 00:00 2024-09-14T03:26:57.395 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-14T03:26:57.401 INFO:teuthology.orchestra.run.smithi195.stdout:(60/114): python3-markupsafe-1.1.1-12.el9.x86_6 837 kB/s | 35 kB 00:00 2024-09-14T03:26:57.443 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-14T03:26:57.484 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-14T03:26:57.529 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-14T03:26:57.573 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-14T03:26:57.698 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-14T03:26:57.740 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-14T03:26:57.779 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-14T03:26:57.879 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-14T03:26:57.919 INFO:teuthology.orchestra.run.smithi195.stdout:(61/114): openblas-openmp-0.3.26-2.el9.x86_64.r 3.8 MB/s | 4.9 MB 00:01 2024-09-14T03:26:57.933 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-14T03:26:58.010 INFO:teuthology.orchestra.run.smithi195.stdout:(62/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 2.7 MB/s | 250 kB 00:00 2024-09-14T03:26:58.102 INFO:teuthology.orchestra.run.smithi195.stdout:(63/114): python3-oauthlib-3.1.1-5.el9.noarch.r 2.4 MB/s | 222 kB 00:00 2024-09-14T03:26:58.161 INFO:teuthology.orchestra.run.smithi195.stdout:(64/114): python3-packaging-20.9-5.el9.noarch.r 1.3 MB/s | 77 kB 00:00 2024-09-14T03:26:58.195 INFO:teuthology.orchestra.run.smithi195.stdout:(65/114): python3-pluggy-0.13.1-7.el9.noarch.rp 1.2 MB/s | 41 kB 00:00 2024-09-14T03:26:58.230 INFO:teuthology.orchestra.run.smithi195.stdout:(66/114): python3-prettytable-0.7.2-27.el9.noar 1.2 MB/s | 42 kB 00:00 2024-09-14T03:26:58.381 INFO:teuthology.orchestra.run.smithi195.stdout:(67/114): python3-py-1.10.0-6.el9.noarch.rpm 3.1 MB/s | 477 kB 00:00 2024-09-14T03:26:58.454 INFO:teuthology.orchestra.run.smithi195.stdout:(68/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.1 MB/s | 159 kB 00:00 2024-09-14T03:26:58.571 INFO:teuthology.orchestra.run.smithi195.stdout:(69/114): python3-babel-2.9.1-2.el9.noarch.rpm 3.2 MB/s | 6.0 MB 00:01 2024-09-14T03:26:58.621 INFO:teuthology.orchestra.run.smithi195.stdout:(70/114): python3-pyasn1-modules-0.4.8-6.el9.no 1.6 MB/s | 279 kB 00:00 2024-09-14T03:26:58.655 INFO:teuthology.orchestra.run.smithi195.stdout:(71/114): python3-pytz-2021.1-5.el9.noarch.rpm 1.5 MB/s | 51 kB 00:00 2024-09-14T03:26:58.663 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-14T03:26:58.688 INFO:teuthology.orchestra.run.smithi195.stdout:(72/114): python3-requests-oauthlib-1.3.0-12.el 1.6 MB/s | 54 kB 00:00 2024-09-14T03:26:58.709 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-14T03:26:58.743 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-14T03:26:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-14T03:26:58.789 INFO:teuthology.orchestra.run.smithi195.stdout:(73/114): python3-pytest-6.2.2-7.el9.noarch.rpm 2.3 MB/s | 519 kB 00:00 2024-09-14T03:26:58.822 INFO:teuthology.orchestra.run.smithi195.stdout:(74/114): python3-toml-0.10.2-6.el9.noarch.rpm 1.2 MB/s | 42 kB 00:00 2024-09-14T03:26:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-14T03:26:58.939 INFO:teuthology.orchestra.run.smithi195.stdout:(75/114): socat-1.7.4.1-6.el9.x86_64.rpm 2.5 MB/s | 304 kB 00:00 2024-09-14T03:26:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-14T03:26:58.986 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-14T03:26:58.989 INFO:teuthology.orchestra.run.smithi195.stdout:(76/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-14T03:26:59.073 INFO:teuthology.orchestra.run.smithi195.stdout:(77/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 3.0 MB/s | 5.1 MB 00:01 2024-09-14T03:26:59.115 INFO:teuthology.orchestra.run.smithi195.stdout:(78/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 2.4 MB/s | 308 kB 00:00 2024-09-14T03:26:59.140 INFO:teuthology.orchestra.run.smithi195.stdout:(79/114): libunwind-1.6.2-1.el9.x86_64.rpm 2.9 MB/s | 67 kB 00:00 2024-09-14T03:26:59.157 INFO:teuthology.orchestra.run.smithi195.stdout:(80/114): liboath-2.6.7-2.el9.x86_64.rpm 594 kB/s | 49 kB 00:00 2024-09-14T03:26:59.169 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-14T03:26:59.174 INFO:teuthology.orchestra.run.smithi195.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 2.6 MB/s | 43 kB 00:00 2024-09-14T03:26:59.191 INFO:teuthology.orchestra.run.smithi195.stdout:(82/114): python3-cachetools-4.2.4-1.el9.noarch 1.9 MB/s | 32 kB 00:00 2024-09-14T03:26:59.193 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-14T03:26:59.216 INFO:teuthology.orchestra.run.smithi195.stdout:(83/114): python3-certifi-2023.05.07-4.el9.noar 565 kB/s | 14 kB 00:00 2024-09-14T03:26:59.250 INFO:teuthology.orchestra.run.smithi195.stdout:(84/114): python3-asyncssh-2.13.2-5.el9.noarch. 4.9 MB/s | 548 kB 00:00 2024-09-14T03:26:59.276 INFO:teuthology.orchestra.run.smithi195.stdout:(85/114): python3-cheroot-8.6.0-4.el9.noarch.rp 2.9 MB/s | 172 kB 00:00 2024-09-14T03:26:59.278 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-14T03:26:59.317 INFO:teuthology.orchestra.run.smithi195.stdout:(86/114): python3-cherrypy-18.6.1-2.el9.noarch. 5.2 MB/s | 358 kB 00:00 2024-09-14T03:26:59.343 INFO:teuthology.orchestra.run.smithi195.stdout:(87/114): python3-google-auth-2.34.0-1.el9.noar 3.2 MB/s | 220 kB 00:00 2024-09-14T03:26:59.368 INFO:teuthology.orchestra.run.smithi195.stdout:(88/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 213 kB/s | 11 kB 00:00 2024-09-14T03:26:59.393 INFO:teuthology.orchestra.run.smithi195.stdout:(89/114): python3-jaraco-classes-3.2.1-5.el9.no 352 kB/s | 18 kB 00:00 2024-09-14T03:26:59.418 INFO:teuthology.orchestra.run.smithi195.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 461 kB/s | 23 kB 00:00 2024-09-14T03:26:59.444 INFO:teuthology.orchestra.run.smithi195.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 388 kB/s | 19 kB 00:00 2024-09-14T03:26:59.469 INFO:teuthology.orchestra.run.smithi195.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 391 kB/s | 20 kB 00:00 2024-09-14T03:26:59.495 INFO:teuthology.orchestra.run.smithi195.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 180 kB/s | 9.0 kB 00:00 2024-09-14T03:26:59.520 INFO:teuthology.orchestra.run.smithi195.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 816 kB/s | 41 kB 00:00 2024-09-14T03:26:59.545 INFO:teuthology.orchestra.run.smithi195.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 1.8 MB/s | 46 kB 00:00 2024-09-14T03:26:59.570 INFO:teuthology.orchestra.run.smithi195.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 3.1 MB/s | 79 kB 00:00 2024-09-14T03:26:59.604 INFO:teuthology.orchestra.run.smithi195.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 1.7 MB/s | 58 kB 00:00 2024-09-14T03:26:59.638 INFO:teuthology.orchestra.run.smithi195.stdout:(98/114): python3-pecan-1.4.2-3.el9.noarch.rpm 8.0 MB/s | 272 kB 00:00 2024-09-14T03:26:59.663 INFO:teuthology.orchestra.run.smithi195.stdout:(99/114): python3-portend-3.1.0-2.el9.noarch.rp 662 kB/s | 16 kB 00:00 2024-09-14T03:26:59.688 INFO:teuthology.orchestra.run.smithi195.stdout:(100/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 3.5 MB/s | 90 kB 00:00 2024-09-14T03:26:59.722 INFO:teuthology.orchestra.run.smithi195.stdout:(101/114): python3-kubernetes-26.1.0-2.el9.noar 4.5 MB/s | 1.0 MB 00:00 2024-09-14T03:26:59.747 INFO:teuthology.orchestra.run.smithi195.stdout:(102/114): python3-repoze-lru-0.7-16.el9.noarch 524 kB/s | 31 kB 00:00 2024-09-14T03:26:59.781 INFO:teuthology.orchestra.run.smithi195.stdout:(103/114): python3-rsa-4.9-2.el9.noarch.rpm 1.7 MB/s | 59 kB 00:00 2024-09-14T03:26:59.807 INFO:teuthology.orchestra.run.smithi195.stdout:(104/114): python3-routes-2.5.1-5.el9.noarch.rp 2.2 MB/s | 188 kB 00:00 2024-09-14T03:26:59.832 INFO:teuthology.orchestra.run.smithi195.stdout:(105/114): python3-tempora-5.0.0-2.el9.noarch.r 713 kB/s | 36 kB 00:00 2024-09-14T03:26:59.857 INFO:teuthology.orchestra.run.smithi195.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 1.5 MB/s | 74 kB 00:00 2024-09-14T03:26:59.891 INFO:teuthology.orchestra.run.smithi195.stdout:(107/114): python3-websocket-client-1.2.3-2.el9 2.6 MB/s | 90 kB 00:00 2024-09-14T03:26:59.916 INFO:teuthology.orchestra.run.smithi195.stdout:(108/114): python3-webob-1.8.8-2.el9.noarch.rpm 2.7 MB/s | 230 kB 00:00 2024-09-14T03:26:59.933 INFO:teuthology.orchestra.run.smithi195.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 1.2 MB/s | 20 kB 00:00 2024-09-14T03:26:59.950 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-14T03:26:59.966 INFO:teuthology.orchestra.run.smithi195.stdout:(110/114): python3-werkzeug-2.0.3-3.el9.1.noarc 5.5 MB/s | 427 kB 00:00 2024-09-14T03:27:00.017 INFO:teuthology.orchestra.run.smithi195.stdout:(111/114): thrift-0.15.0-2.el9.x86_64.rpm 19 MB/s | 1.6 MB 00:00 2024-09-14T03:27:00.066 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-14T03:27:00.267 INFO:teuthology.orchestra.run.smithi195.stdout:(112/114): librbd1-17.2.7-1629.geb16b97b.el9.x8 12 MB/s | 2.9 MB 00:00 2024-09-14T03:27:00.367 INFO:teuthology.orchestra.run.smithi195.stdout:(113/114): librados2-17.2.7-1629.geb16b97b.el9. 7.7 MB/s | 3.1 MB 00:00 2024-09-14T03:27:01.349 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-14T03:27:01.439 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-14T03:27:01.730 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-14T03:27:01.767 INFO:teuthology.orchestra.run.smithi110.stdout: Upgrading : librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 75/116 2024-09-14T03:27:01.840 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 75/116 2024-09-14T03:27:01.880 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 76/116 2024-09-14T03:27:01.929 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-14T03:27:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-14T03:27:02.171 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-14T03:27:02.226 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-14T03:27:02.297 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-14T03:27:02.384 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-14T03:27:02.523 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-14T03:27:02.564 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-14T03:27:02.594 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-14T03:27:02.627 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-14T03:27:02.668 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-14T03:27:02.696 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-14T03:27:03.140 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 89/116 2024-09-14T03:27:03.168 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 89/116 2024-09-14T03:27:03.198 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.e 90/116 2024-09-14T03:27:03.272 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b. 91/116 2024-09-14T03:27:03.301 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-ceph-common-2:17.2.7-1629.geb16b97b.el9. 92/116 2024-09-14T03:27:03.435 INFO:teuthology.orchestra.run.smithi195.stdout:(114/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 3.4 MB/s | 16 MB 00:04 2024-09-14T03:27:03.440 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:27:03.440 INFO:teuthology.orchestra.run.smithi195.stdout:Total 13 MB/s | 190 MB 00:14 2024-09-14T03:27:04.171 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-09-14T03:27:04.268 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-09-14T03:27:04.268 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-09-14T03:27:05.687 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:27:05.723 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:27:06.015 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-09-14T03:27:06.015 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-09-14T03:27:06.238 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:27:06.276 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 94/116 2024-09-14T03:27:06.368 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 94/116 2024-09-14T03:27:06.368 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:27:06.368 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:27:06.369 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:06.403 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 95/116 2024-09-14T03:27:10.686 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-09-14T03:27:10.778 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-14T03:27:10.803 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-14T03:27:10.871 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-14T03:27:11.181 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-14T03:27:11.368 INFO:teuthology.orchestra.run.smithi195.stdout: Upgrading : librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 5/116 2024-09-14T03:27:11.594 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 5/116 2024-09-14T03:27:11.622 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:27:11.698 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:27:11.745 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 7/116 2024-09-14T03:27:11.777 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-14T03:27:12.216 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-14T03:27:12.250 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:27:12.324 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:27:12.351 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:27:12.451 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:27:12.477 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 12/116 2024-09-14T03:27:12.604 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 12/116 2024-09-14T03:27:12.640 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-14T03:27:12.680 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-14T03:27:12.710 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-14T03:27:12.751 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-14T03:27:12.825 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-14T03:27:12.886 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-14T03:27:12.926 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-14T03:27:12.953 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-14T03:27:13.023 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-14T03:27:13.076 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-14T03:27:13.112 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-14T03:27:13.149 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-14T03:27:13.193 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-14T03:27:13.245 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el 26/116 2024-09-14T03:27:13.320 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_6 27/116 2024-09-14T03:27:13.462 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-14T03:27:13.515 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-14T03:27:14.040 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-14T03:27:14.129 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-14T03:27:14.173 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-14T03:27:14.207 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-14T03:27:14.242 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-14T03:27:14.280 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-14T03:27:14.316 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 36/116 2024-09-14T03:27:14.355 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_6 37/116 2024-09-14T03:27:14.385 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-14T03:27:14.453 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-14T03:27:14.519 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-14T03:27:14.592 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-14T03:27:14.623 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-14T03:27:14.659 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-14T03:27:14.710 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-14T03:27:14.754 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-14T03:27:14.797 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-14T03:27:14.934 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-14T03:27:14.982 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-14T03:27:15.023 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-14T03:27:15.127 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-14T03:27:15.166 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-14T03:27:15.213 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-14T03:27:15.251 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-14T03:27:15.296 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-14T03:27:15.419 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-14T03:27:15.454 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-14T03:27:15.493 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-14T03:27:15.600 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-14T03:27:15.656 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-14T03:27:16.389 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-14T03:27:16.436 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-14T03:27:16.470 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-14T03:27:16.503 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-14T03:27:16.605 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-14T03:27:16.663 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-14T03:27:16.706 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-14T03:27:16.890 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-14T03:27:17.005 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-14T03:27:17.250 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-14T03:27:17.889 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 95/116 2024-09-14T03:27:17.889 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /sys 2024-09-14T03:27:17.889 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /proc 2024-09-14T03:27:17.890 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /mnt 2024-09-14T03:27:17.890 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /var/tmp 2024-09-14T03:27:17.890 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /home 2024-09-14T03:27:17.890 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /root 2024-09-14T03:27:17.890 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /tmp 2024-09-14T03:27:17.890 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:17.918 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-14T03:27:17.958 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 96/116 2024-09-14T03:27:18.036 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-14T03:27:18.963 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 96/116 2024-09-14T03:27:19.004 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 97/116 2024-09-14T03:27:19.319 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-14T03:27:19.419 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-14T03:27:19.720 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-14T03:27:19.772 INFO:teuthology.orchestra.run.smithi195.stdout: Upgrading : librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 75/116 2024-09-14T03:27:19.845 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 75/116 2024-09-14T03:27:19.894 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 76/116 2024-09-14T03:27:19.943 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-14T03:27:19.959 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 97/116 2024-09-14T03:27:19.988 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 98/116 2024-09-14T03:27:19.998 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-14T03:27:20.088 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 98/116 2024-09-14T03:27:20.183 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-14T03:27:20.231 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-14T03:27:20.249 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el 99/116 2024-09-14T03:27:20.273 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 100/116 2024-09-14T03:27:20.297 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 100/116 2024-09-14T03:27:20.298 INFO:teuthology.orchestra.run.smithi110.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:20.298 INFO:teuthology.orchestra.run.smithi110.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:27:20.298 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:27:20.298 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:27:20.298 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:20.318 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-14T03:27:20.340 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 101/116 2024-09-14T03:27:20.406 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-14T03:27:20.524 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 101/116 2024-09-14T03:27:20.545 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-14T03:27:20.558 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 102/116 2024-09-14T03:27:20.581 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 102/116 2024-09-14T03:27:20.581 INFO:teuthology.orchestra.run.smithi110.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:20.581 INFO:teuthology.orchestra.run.smithi110.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:27:20.582 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:27:20.582 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:27:20.582 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:20.594 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-14T03:27:20.624 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-14T03:27:20.677 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-14T03:27:20.731 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-14T03:27:20.759 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-14T03:27:20.967 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 103/116 2024-09-14T03:27:20.991 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 103/116 2024-09-14T03:27:20.991 INFO:teuthology.orchestra.run.smithi110.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:20.991 INFO:teuthology.orchestra.run.smithi110.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:27:20.991 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:27:20.991 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:27:20.991 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:21.169 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 89/116 2024-09-14T03:27:21.198 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 89/116 2024-09-14T03:27:21.228 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.e 90/116 2024-09-14T03:27:21.301 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b. 91/116 2024-09-14T03:27:21.331 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-ceph-common-2:17.2.7-1629.geb16b97b.el9. 92/116 2024-09-14T03:27:22.312 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 104/116 2024-09-14T03:27:22.336 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 104/116 2024-09-14T03:27:22.336 INFO:teuthology.orchestra.run.smithi110.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:22.336 INFO:teuthology.orchestra.run.smithi110.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:27:22.336 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:27:22.337 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:27:22.337 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 105/116 2024-09-14T03:27:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 106/116 2024-09-14T03:27:23.691 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:27:23.705 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 106/116 2024-09-14T03:27:23.705 INFO:teuthology.orchestra.run.smithi110.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:23.705 INFO:teuthology.orchestra.run.smithi110.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:27:23.705 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:27:23.705 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:27:23.706 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:23.730 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:27:23.748 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1629.geb16b 107/116 2024-09-14T03:27:23.769 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1629.geb16b 107/116 2024-09-14T03:27:23.769 INFO:teuthology.orchestra.run.smithi110.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:23.769 INFO:teuthology.orchestra.run.smithi110.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:27:23.770 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:24.054 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 108/116 2024-09-14T03:27:24.078 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 108/116 2024-09-14T03:27:24.078 INFO:teuthology.orchestra.run.smithi110.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:24.079 INFO:teuthology.orchestra.run.smithi110.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:27:24.079 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:27:24.079 INFO:teuthology.orchestra.run.smithi110.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:27:24.079 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:24.245 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:27:24.291 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 94/116 2024-09-14T03:27:24.390 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 94/116 2024-09-14T03:27:24.390 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:27:24.390 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:27:24.390 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:24.434 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 95/116 2024-09-14T03:27:28.493 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 109/116 2024-09-14T03:27:28.525 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 110/116 2024-09-14T03:27:28.566 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 111/116 2024-09-14T03:27:28.684 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 112/116 2024-09-14T03:27:28.715 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-14T03:27:28.738 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_ 114/116 2024-09-14T03:27:28.739 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-14T03:27:28.831 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-14T03:27:28.831 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:27:33.823 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:27:33.823 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 1/116 2024-09-14T03:27:33.823 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 2/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 3/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 4/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1629.geb16b 5/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 7/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 8/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 9/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 12/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_ 13/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 14/116 2024-09-14T03:27:33.824 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 15/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_6 16/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 17/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 18/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el 19/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-ceph-common-2:17.2.7-1629.geb16b97b.el9. 20/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_6 21/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 22/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 23/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 24/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 25/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 26/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 27/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b. 28/116 2024-09-14T03:27:33.825 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 29/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 30/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 31/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el 32/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 33/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.e 34/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 35/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-14T03:27:33.826 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-14T03:27:33.827 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-14T03:27:33.828 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-14T03:27:33.829 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-14T03:27:33.830 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-14T03:27:33.831 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-14T03:27:33.832 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-14T03:27:33.832 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-14T03:27:33.832 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 113/116 2024-09-14T03:27:33.832 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-14T03:27:33.832 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 115/116 2024-09-14T03:27:35.800 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 95/116 2024-09-14T03:27:35.800 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /sys 2024-09-14T03:27:35.801 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /proc 2024-09-14T03:27:35.801 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /mnt 2024-09-14T03:27:35.801 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /var/tmp 2024-09-14T03:27:35.801 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /home 2024-09-14T03:27:35.801 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /root 2024-09-14T03:27:35.801 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /tmp 2024-09-14T03:27:35.801 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:35.879 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 96/116 2024-09-14T03:27:36.044 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout:Upgraded: 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:36.045 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-immutable-object-cache-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.046 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.047 INFO:teuthology.orchestra.run.smithi110.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T03:27:36.048 INFO:teuthology.orchestra.run.smithi110.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:27:36.049 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T03:27:36.050 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T03:27:36.051 INFO:teuthology.orchestra.run.smithi110.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-14T03:27:36.052 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:27:36.053 INFO:teuthology.orchestra.run.smithi110.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:27:36.054 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2024-09-14T03:27:36.388 DEBUG:teuthology.parallel:result is None 2024-09-14T03:27:36.874 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 96/116 2024-09-14T03:27:36.917 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 97/116 2024-09-14T03:27:37.871 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 97/116 2024-09-14T03:27:37.900 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 98/116 2024-09-14T03:27:38.000 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 98/116 2024-09-14T03:27:38.145 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el 99/116 2024-09-14T03:27:38.185 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 100/116 2024-09-14T03:27:38.209 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 100/116 2024-09-14T03:27:38.209 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:38.209 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:27:38.209 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:27:38.209 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:27:38.209 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:38.261 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 101/116 2024-09-14T03:27:38.446 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 101/116 2024-09-14T03:27:38.487 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 102/116 2024-09-14T03:27:38.511 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 102/116 2024-09-14T03:27:38.511 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:38.511 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:27:38.511 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:27:38.511 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:27:38.511 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:38.896 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 103/116 2024-09-14T03:27:38.919 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 103/116 2024-09-14T03:27:38.919 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:38.919 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:27:38.919 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:27:38.919 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:27:38.919 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:40.247 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 104/116 2024-09-14T03:27:40.273 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 104/116 2024-09-14T03:27:40.273 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:40.273 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:27:40.274 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:27:40.274 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:27:40.274 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:41.524 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 105/116 2024-09-14T03:27:41.556 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 106/116 2024-09-14T03:27:41.634 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 106/116 2024-09-14T03:27:41.634 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:41.634 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:27:41.634 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:27:41.635 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:27:41.635 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:41.674 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1629.geb16b 107/116 2024-09-14T03:27:41.697 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1629.geb16b 107/116 2024-09-14T03:27:41.697 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:41.697 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:27:41.698 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:41.984 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 108/116 2024-09-14T03:27:42.007 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 108/116 2024-09-14T03:27:42.007 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:27:42.007 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:27:42.007 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:27:42.008 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:27:42.008 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:46.422 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 109/116 2024-09-14T03:27:46.454 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 110/116 2024-09-14T03:27:46.494 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 111/116 2024-09-14T03:27:46.613 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 112/116 2024-09-14T03:27:46.643 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-14T03:27:46.667 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_ 114/116 2024-09-14T03:27:46.667 INFO:teuthology.orchestra.run.smithi195.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-14T03:27:46.742 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-14T03:27:46.742 INFO:teuthology.orchestra.run.smithi195.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 1/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 2/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 3/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 4/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1629.geb16b 5/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 7/116 2024-09-14T03:27:51.766 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 8/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 9/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 12/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_ 13/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 14/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 15/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_6 16/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 17/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 18/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el 19/116 2024-09-14T03:27:51.767 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-ceph-common-2:17.2.7-1629.geb16b97b.el9. 20/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_6 21/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 22/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 23/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 24/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 25/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 26/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 27/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b. 28/116 2024-09-14T03:27:51.768 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 29/116 2024-09-14T03:27:51.779 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 30/116 2024-09-14T03:27:51.779 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 31/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el 32/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 33/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.e 34/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 35/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-14T03:27:51.780 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-14T03:27:51.781 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-14T03:27:51.782 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-14T03:27:51.783 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-14T03:27:51.784 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-14T03:27:51.785 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 113/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-14T03:27:51.786 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 115/116 2024-09-14T03:27:54.065 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:27:54.065 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:54.065 INFO:teuthology.orchestra.run.smithi195.stdout:Upgraded: 2024-09-14T03:27:54.065 INFO:teuthology.orchestra.run.smithi195.stdout: librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.065 INFO:teuthology.orchestra.run.smithi195.stdout: librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.065 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-09-14T03:27:54.065 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-immutable-object-cache-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:54.066 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-14T03:27:54.067 INFO:teuthology.orchestra.run.smithi195.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:27:54.068 INFO:teuthology.orchestra.run.smithi195.stdout: librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.069 INFO:teuthology.orchestra.run.smithi195.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:27:54.070 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T03:27:54.071 INFO:teuthology.orchestra.run.smithi195.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T03:27:54.072 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T03:27:54.073 INFO:teuthology.orchestra.run.smithi195.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:27:54.074 INFO:teuthology.orchestra.run.smithi195.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:27:54.075 INFO:teuthology.orchestra.run.smithi195.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:27:54.075 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:27:54.075 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-09-14T03:27:54.402 DEBUG:teuthology.parallel:result is None 2024-09-14T03:27:54.403 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:27:54.403 INFO:teuthology.packaging:ref: None 2024-09-14T03:27:54.403 INFO:teuthology.packaging:tag: None 2024-09-14T03:27:54.403 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:27:54.403 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:27:54.403 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-14T03:27:54.645 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:27:54.671 INFO:teuthology.orchestra.run.smithi110.stdout:17.2.7-1629.geb16b97b.el9 2024-09-14T03:27:54.672 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1629.geb16b97b.el9 2024-09-14T03:27:54.672 INFO:teuthology.task.install:The correct ceph version 17.2.7-1629.geb16b97b is installed. 2024-09-14T03:27:54.674 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:27:54.674 INFO:teuthology.packaging:ref: None 2024-09-14T03:27:54.674 INFO:teuthology.packaging:tag: None 2024-09-14T03:27:54.674 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:27:54.674 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:27:54.675 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-14T03:27:54.904 DEBUG:teuthology.orchestra.run.smithi195:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:27:54.927 INFO:teuthology.orchestra.run.smithi195.stdout:17.2.7-1629.geb16b97b.el9 2024-09-14T03:27:54.928 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1629.geb16b97b.el9 2024-09-14T03:27:54.928 INFO:teuthology.task.install:The correct ceph version 17.2.7-1629.geb16b97b is installed. 2024-09-14T03:27:54.929 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-14T03:27:54.930 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:27:54.930 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:27:54.963 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:27:54.964 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:27:54.996 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-14T03:27:54.996 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:27:54.996 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:27:55.031 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:27:55.098 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:27:55.099 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:27:55.125 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:27:55.191 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-14T03:27:55.191 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:27:55.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:27:55.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:27:55.284 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:27:55.284 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:27:55.309 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:27:55.375 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-14T03:27:55.376 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:27:55.376 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:27:55.400 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:27:55.465 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:27:55.465 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:27:55.491 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:27:55.557 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:27:55.568 INFO:teuthology.task.print:**** done install task... 2024-09-14T03:27:55.568 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:27:55.617 INFO:teuthology.task.print:**** done start installing quincy cephadm ... 2024-09-14T03:27:55.618 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-14T03:27:55.756 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:27:55.756 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-14T03:27:55.757 INFO:tasks.cephadm:Cluster fsid is 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:27:55.757 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-14T03:27:55.757 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.110', 'mon.c': '[v2:172.21.15.110:3301,v1:172.21.15.110:6790]', 'mon.b': '172.21.15.195'} 2024-09-14T03:27:55.757 INFO:tasks.cephadm:First mon is mon.a on smithi110 2024-09-14T03:27:55.757 INFO:tasks.cephadm:First mgr is y 2024-09-14T03:27:55.757 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-14T03:27:55.757 DEBUG:teuthology.orchestra.run.smithi110:> sudo hostname $(hostname -s) 2024-09-14T03:27:55.783 DEBUG:teuthology.orchestra.run.smithi195:> sudo hostname $(hostname -s) 2024-09-14T03:27:55.809 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-14T03:27:55.809 DEBUG:teuthology.orchestra.run.smithi110:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-14T03:27:56.062 INFO:teuthology.orchestra.run.smithi110.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 14 03:27 /home/ubuntu/cephtest/cephadm 2024-09-14T03:27:56.062 DEBUG:teuthology.orchestra.run.smithi195:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-14T03:27:56.192 INFO:teuthology.orchestra.run.smithi195.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 14 03:27 /home/ubuntu/cephtest/cephadm 2024-09-14T03:27:56.192 DEBUG:teuthology.orchestra.run.smithi110:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-14T03:27:56.208 DEBUG:teuthology.orchestra.run.smithi195:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-14T03:27:56.236 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-14T03:27:56.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-14T03:27:56.250 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-14T03:27:57.879 INFO:teuthology.orchestra.run.smithi110.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-14T03:27:57.945 INFO:teuthology.orchestra.run.smithi195.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-14T03:28:23.835 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:28:23.835 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph_version": "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)", 2024-09-14T03:28:23.835 INFO:teuthology.orchestra.run.smithi110.stdout: "image_id": "ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446", 2024-09-14T03:28:23.835 INFO:teuthology.orchestra.run.smithi110.stdout: "repo_digests": [ 2024-09-14T03:28:23.836 INFO:teuthology.orchestra.run.smithi110.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6" 2024-09-14T03:28:23.836 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2024-09-14T03:28:23.836 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:28:30.589 INFO:teuthology.orchestra.run.smithi195.stdout:{ 2024-09-14T03:28:30.590 INFO:teuthology.orchestra.run.smithi195.stdout: "ceph_version": "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)", 2024-09-14T03:28:30.590 INFO:teuthology.orchestra.run.smithi195.stdout: "image_id": "ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446", 2024-09-14T03:28:30.590 INFO:teuthology.orchestra.run.smithi195.stdout: "repo_digests": [ 2024-09-14T03:28:30.590 INFO:teuthology.orchestra.run.smithi195.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6" 2024-09-14T03:28:30.590 INFO:teuthology.orchestra.run.smithi195.stdout: ] 2024-09-14T03:28:30.591 INFO:teuthology.orchestra.run.smithi195.stdout:} 2024-09-14T03:28:30.608 DEBUG:teuthology.orchestra.run.smithi110:> sudo mkdir -p /etc/ceph 2024-09-14T03:28:30.641 DEBUG:teuthology.orchestra.run.smithi195:> sudo mkdir -p /etc/ceph 2024-09-14T03:28:30.669 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 777 /etc/ceph 2024-09-14T03:28:30.709 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 777 /etc/ceph 2024-09-14T03:28:30.734 INFO:tasks.cephadm:Writing seed config... 2024-09-14T03:28:30.735 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-14T03:28:30.736 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-14T03:28:30.737 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:28:30.737 DEBUG:teuthology.orchestra.run.smithi110:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-14T03:28:30.767 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 5517b266-7249-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-14T03:28:30.767 DEBUG:teuthology.orchestra.run.smithi110:mon.a> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a.service 2024-09-14T03:28:30.810 DEBUG:teuthology.orchestra.run.smithi110:mgr.y> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service 2024-09-14T03:28:30.853 INFO:tasks.cephadm:Bootstrapping... 2024-09-14T03:28:30.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 5517b266-7249-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.110 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:28:31.062 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:28:31.062 INFO:teuthology.orchestra.run.smithi110.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '5517b266-7249-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.110', '--skip-admin-label'] 2024-09-14T03:28:31.086 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:28:31.086 INFO:teuthology.orchestra.run.smithi110.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-14T03:28:31.087 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying podman|docker is present... 2024-09-14T03:28:31.108 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:28:31.109 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying lvm2 is present... 2024-09-14T03:28:31.109 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying time synchronization is in place... 2024-09-14T03:28:31.118 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-14T03:28:31.118 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-14T03:28:31.126 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-14T03:28:31.126 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2024-09-14T03:28:31.134 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout enabled 2024-09-14T03:28:31.142 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout active 2024-09-14T03:28:31.142 INFO:teuthology.orchestra.run.smithi110.stdout:Unit chronyd.service is enabled and running 2024-09-14T03:28:31.142 INFO:teuthology.orchestra.run.smithi110.stdout:Repeating the final host check... 2024-09-14T03:28:31.165 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:28:31.165 INFO:teuthology.orchestra.run.smithi110.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-14T03:28:31.165 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl is present 2024-09-14T03:28:31.165 INFO:teuthology.orchestra.run.smithi110.stdout:lvcreate is present 2024-09-14T03:28:31.172 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-14T03:28:31.172 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-14T03:28:31.179 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-14T03:28:31.179 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2024-09-14T03:28:31.187 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout enabled 2024-09-14T03:28:31.195 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout active 2024-09-14T03:28:31.195 INFO:teuthology.orchestra.run.smithi110.stdout:Unit chronyd.service is enabled and running 2024-09-14T03:28:31.195 INFO:teuthology.orchestra.run.smithi110.stdout:Host looks OK 2024-09-14T03:28:31.195 INFO:teuthology.orchestra.run.smithi110.stdout:Cluster fsid: 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:28:31.195 INFO:teuthology.orchestra.run.smithi110.stdout:Acquiring lock 140665555739552 on /run/cephadm/5517b266-7249-11ef-bceb-c7b262605968.lock 2024-09-14T03:28:31.196 INFO:teuthology.orchestra.run.smithi110.stdout:Lock 140665555739552 acquired on /run/cephadm/5517b266-7249-11ef-bceb-c7b262605968.lock 2024-09-14T03:28:31.196 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying IP 172.21.15.110 port 3300 ... 2024-09-14T03:28:31.197 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying IP 172.21.15.110 port 6789 ... 2024-09-14T03:28:31.197 INFO:teuthology.orchestra.run.smithi110.stdout:Base mon IP(s) is [172.21.15.110:3300, 172.21.15.110:6789], mon addrv is [v2:172.21.15.110:3300,v1:172.21.15.110:6789] 2024-09-14T03:28:31.199 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.110 metric 100 2024-09-14T03:28:31.199 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.110 metric 100 2024-09-14T03:28:31.203 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-14T03:28:31.203 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-14T03:28:31.203 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-14T03:28:31.206 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-14T03:28:31.206 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-14T03:28:31.206 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-14T03:28:31.206 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-14T03:28:31.206 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:cc35/64 scope link noprefixroute 2024-09-14T03:28:31.206 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-14T03:28:31.207 INFO:teuthology.orchestra.run.smithi110.stdout:Mon IP `172.21.15.110` is in CIDR network `172.21.0.0/20` 2024-09-14T03:28:31.207 INFO:teuthology.orchestra.run.smithi110.stdout:Mon IP `172.21.15.110` is in CIDR network `172.21.0.0/20` 2024-09-14T03:28:31.207 INFO:teuthology.orchestra.run.smithi110.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-14T03:28:31.207 INFO:teuthology.orchestra.run.smithi110.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-14T03:28:31.208 INFO:teuthology.orchestra.run.smithi110.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-14T03:28:31.686 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stdout ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 2024-09-14T03:28:31.686 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-14T03:28:31.686 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Getting image source signatures 2024-09-14T03:28:31.686 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Copying blob sha256:9b559dbc30e0607f8b5bc9e421c2d3b7f5b89b8ae7a0518eaf6d2b64276fd6bc 2024-09-14T03:28:31.686 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-14T03:28:31.686 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Copying config sha256:ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 2024-09-14T03:28:31.686 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-14T03:28:32.640 INFO:teuthology.orchestra.run.smithi110.stdout:ceph: stdout ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable) 2024-09-14T03:28:32.641 INFO:teuthology.orchestra.run.smithi110.stdout:Ceph version: ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable) 2024-09-14T03:28:32.641 INFO:teuthology.orchestra.run.smithi110.stdout:Extracting ceph user uid/gid from container image... 2024-09-14T03:28:33.547 INFO:teuthology.orchestra.run.smithi110.stdout:stat: stdout 167 167 2024-09-14T03:28:33.547 INFO:teuthology.orchestra.run.smithi110.stdout:Creating initial keys... 2024-09-14T03:28:34.445 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph-authtool: stdout AQDhAuVm+zFNNBAAhsRN6yXqs1QpDWfaTLIn7g== 2024-09-14T03:28:35.360 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph-authtool: stdout AQDiAuVm4J+VLBAAY0EscFRLi7osWRJVp4GQ7A== 2024-09-14T03:28:36.197 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph-authtool: stdout AQDjAuVmPQuAJRAAjAEAlviWNWKiXd6TNnzgIg== 2024-09-14T03:28:36.198 INFO:teuthology.orchestra.run.smithi110.stdout:Creating initial monmap... 2024-09-14T03:28:37.111 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-14T03:28:37.111 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-14T03:28:37.111 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:28:37.111 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-14T03:28:37.111 INFO:teuthology.orchestra.run.smithi110.stdout:monmaptool for a [v2:172.21.15.110:3300,v1:172.21.15.110:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-14T03:28:37.112 INFO:teuthology.orchestra.run.smithi110.stdout:setting min_mon_release = octopus 2024-09-14T03:28:37.112 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: set fsid to 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:28:37.112 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-14T03:28:37.112 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:28:37.112 INFO:teuthology.orchestra.run.smithi110.stdout:Creating mon... 2024-09-14T03:28:38.041 INFO:teuthology.orchestra.run.smithi110.stdout:create mon.a on 2024-09-14T03:28:38.260 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-14T03:28:38.406 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-14T03:28:38.557 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-5517b266-7249-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-5517b266-7249-11ef-bceb-c7b262605968.target. 2024-09-14T03:28:38.557 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-5517b266-7249-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-5517b266-7249-11ef-bceb-c7b262605968.target. 2024-09-14T03:28:38.729 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a 2024-09-14T03:28:38.729 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Failed to reset failed state of unit ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-14T03:28:38.896 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-5517b266-7249-11ef-bceb-c7b262605968.target.wants/ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-5517b266-7249-11ef-bceb-c7b262605968@.service. 2024-09-14T03:28:39.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:39 smithi110 ceph-mon[26996]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:28:39.407 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:28:39.407 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout disabled 2024-09-14T03:28:39.415 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:28:39.415 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2024-09-14T03:28:39.415 INFO:teuthology.orchestra.run.smithi110.stdout:firewalld.service is not enabled 2024-09-14T03:28:39.415 INFO:teuthology.orchestra.run.smithi110.stdout:Not possible to enable service . firewalld.service is not available 2024-09-14T03:28:39.415 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mon to start... 2024-09-14T03:28:39.415 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mon... 2024-09-14T03:28:39.649 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:39 smithi110 bash[26978]: e6d51ec608d61ab6621b9b4ab15563ea3c3c345c6eef2a56c445e9a6d25df23e 2024-09-14T03:28:39.650 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:39 smithi110 systemd[1]: Started Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:28:40.328 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout cluster: 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout id: 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout services: 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.507583s) 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout data: 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-14T03:28:40.329 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-14T03:28:40.330 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-14T03:28:40.330 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout pgs: 2024-09-14T03:28:40.330 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:28:40.330 INFO:teuthology.orchestra.run.smithi110.stdout:mon is available 2024-09-14T03:28:40.330 INFO:teuthology.orchestra.run.smithi110.stdout:Assimilating anything we can from ceph.conf... 2024-09-14T03:28:40.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:40 smithi110 ceph-mon[26996]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:28:40.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:40 smithi110 ceph-mon[26996]: monmap e1: 1 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:28:40.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:40 smithi110 ceph-mon[26996]: fsmap 2024-09-14T03:28:40.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:40 smithi110 ceph-mon[26996]: osdmap e1: 0 total, 0 up, 0 in 2024-09-14T03:28:40.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:40 smithi110 ceph-mon[26996]: mgrmap e1: no daemons active 2024-09-14T03:28:40.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:40 smithi110 ceph-mon[26996]: from='client.? 172.21.15.110:0/1343495568' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-14T03:28:41.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:41 smithi110 ceph-mon[26996]: from='client.? 172.21.15.110:0/878199818' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-14T03:28:41.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:41 smithi110 ceph-mon[26996]: from='client.? 172.21.15.110:0/878199818' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-14T03:28:44.812 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [global] 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout fsid = 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.110:3300,v1:172.21.15.110:6789] 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-14T03:28:44.813 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:28:44.814 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [osd] 2024-09-14T03:28:44.814 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-14T03:28:44.814 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-14T03:28:44.814 INFO:teuthology.orchestra.run.smithi110.stdout:Generating new minimal ceph.conf... 2024-09-14T03:28:47.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:47 smithi110 ceph-mon[26996]: from='client.? 172.21.15.110:0/4256025818' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:28:53.430 INFO:teuthology.orchestra.run.smithi110.stdout:Restarting the monitor... 2024-09-14T03:28:53.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:53 smithi110 systemd[1]: Stopping Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:28:53.987 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:53 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[26992]: 2024-09-14T03:28:53.711+0000 7fd9f4c4c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:28:53.988 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:53 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[26992]: 2024-09-14T03:28:53.711+0000 7fd9f4c4c640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-14T03:28:53.988 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:53 smithi110 podman[27241]: 2024-09-14 03:28:53.732103407 +0000 UTC m=+0.133918136 container died e6d51ec608d61ab6621b9b4ab15563ea3c3c345c6eef2a56c445e9a6d25df23e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:28:54.246 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 podman[27241]: 2024-09-14 03:28:54.07012473 +0000 UTC m=+0.471939457 container cleanup e6d51ec608d61ab6621b9b4ab15563ea3c3c345c6eef2a56c445e9a6d25df23e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-14T03:28:54.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 bash[27241]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:28:54.680 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 podman[27253]: 2024-09-14 03:28:54.246050379 +0000 UTC m=+0.510842435 container remove e6d51ec608d61ab6621b9b4ab15563ea3c3c345c6eef2a56c445e9a6d25df23e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T03:28:54.681 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T03:28:54.681 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 systemd[1]: Stopped Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:28:54.681 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 systemd[1]: Starting Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:28:54.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 podman[27346]: 2024-09-14 03:28:54.680181146 +0000 UTC m=+0.109016481 container create c1073676c66c24575c3a0a0a56ec31782bf4290020321ad3b4f76914a50aa092 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 podman[27346]: 2024-09-14 03:28:54.586754298 +0000 UTC m=+0.015589641 image pull ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 podman[27346]: 2024-09-14 03:28:54.890968975 +0000 UTC m=+0.319804319 container init c1073676c66c24575c3a0a0a56ec31782bf4290020321ad3b4f76914a50aa092 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 podman[27346]: 2024-09-14 03:28:54.894110326 +0000 UTC m=+0.322945663 container start c1073676c66c24575c3a0a0a56ec31782bf4290020321ad3b4f76914a50aa092 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable), process ceph-mon, pid 2 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: pidfile_write: ignore empty --pid-file 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: load: jerasure load: lrc 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: RocksDB version: 6.15.5 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Compile date Sep 13 2024 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: DB SUMMARY 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: DB Session ID: ZON8JO19PG928QTGYSLS 2024-09-14T03:28:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:28:54.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:28:54.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-14T03:28:54.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-14T03:28:54.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83708 ; 2024-09-14T03:28:54.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:28:54.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:28:54.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:28:54.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.env: 0x55ad10707440 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.fs: Posix File System 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.info_log: 0x55ad115bcec0 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.statistics: (nil) 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.use_fsync: 0 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:28:54.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.db_log_dir: 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:28:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.write_buffer_manager: 0x55ad1167a780 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:28:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.unordered_write: 0 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.row_cache: None 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.wal_filter: None 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:28:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.preserve_deletes: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:28:54.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:28:54.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_open_files: -1 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Compression algorithms supported: 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: kZSTD supported: 0 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:28:54.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.merge_operator: 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_filter: None 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:28:54.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ad11563a38) 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_type: 0 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_block_index_type: 0 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_shortening: 1 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: hash_index_allow_collision: 1 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: checksum: 1 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: no_block_cache: 0 2024-09-14T03:28:54.958 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache: 0x55ad115991f0 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_options: 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: capacity : 536870912 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: num_shard_bits : 4 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: strict_capacity_limit : 0 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_compressed: (nil) 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: persistent_cache: (nil) 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_size: 4096 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_size_deviation: 10 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_restart_interval: 16 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_block_restart_interval: 1 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: metadata_block_size: 4096 2024-09-14T03:28:54.959 INFO:journalctl@ceph.mon.a.smithi110.stdout: partition_filters: 0 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout: use_delta_encoding: 1 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout: whole_key_filtering: 1 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout: verify_compression: 0 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout: format_version: 4 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout: enable_index_compression: 1 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_align: 0 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compression: NoCompression 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:28:54.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.num_levels: 7 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:28:54.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:28:54.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.arena_block_size: 4194304 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:28:54.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:28:54.964 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.ttl: 2592000 2024-09-14T03:28:54.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284534916725, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-14T03:28:54.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284534917445, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80281, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78523, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72736, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726284534, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "2e53e134-9268-490a-b647-d5e26da85c9a", "db_session_id": "ZON8JO19PG928QTGYSLS"}} 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284534917768, "job": 1, "event": "recovery_finished"} 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55ad115cc700 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: DB pointer 0x55ad1159c000 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** DB Stats ** 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:28:54.967 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: L0 2/0 80.18 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: Sum 2/0 80.18 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:28:54.968 INFO:journalctl@ceph.mon.a.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative compaction: 0.00 GB write, 24.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval compaction: 0.00 GB write, 24.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:28:54.969 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: L0 2/0 80.18 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: Sum 2/0 80.18 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:28:54.970 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative compaction: 0.00 GB write, 24.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: starting mon.a rank 0 at public addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] at bind addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a@-1(???) e1 preinit fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a@-1(???).mds e1 new map 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a@-1(???).mds e1 print_map 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: e1 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:28:54.971 INFO:journalctl@ceph.mon.a.smithi110.stdout: legacy client fscid: -1 2024-09-14T03:28:54.972 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:28:54.972 INFO:journalctl@ceph.mon.a.smithi110.stdout: No filesystems configured 2024-09-14T03:28:54.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:28:54.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:28:54.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:28:54.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:28:54.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-14T03:28:54.997 INFO:teuthology.orchestra.run.smithi110.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-14T03:28:55.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:28:55.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: monmap e1: 1 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:28:55.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: fsmap 2024-09-14T03:28:55.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: osdmap e1: 0 total, 0 up, 0 in 2024-09-14T03:28:55.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 ceph-mon[27365]: mgrmap e1: no daemons active 2024-09-14T03:28:55.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 bash[27346]: c1073676c66c24575c3a0a0a56ec31782bf4290020321ad3b4f76914a50aa092 2024-09-14T03:28:55.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:54 smithi110 systemd[1]: Started Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-14T03:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout:Creating mgr... 2024-09-14T03:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-14T03:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-14T03:28:56.165 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y 2024-09-14T03:28:56.165 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Failed to reset failed state of unit ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-14T03:28:56.318 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-5517b266-7249-11ef-bceb-c7b262605968.target.wants/ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-5517b266-7249-11ef-bceb-c7b262605968@.service. 2024-09-14T03:28:56.649 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:56 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1606070082' entity='client.admin' 2024-09-14T03:28:56.868 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:28:56.868 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout disabled 2024-09-14T03:28:56.877 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:28:56.877 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2024-09-14T03:28:56.877 INFO:teuthology.orchestra.run.smithi110.stdout:firewalld.service is not enabled 2024-09-14T03:28:56.877 INFO:teuthology.orchestra.run.smithi110.stdout:Not possible to enable service . firewalld.service is not available 2024-09-14T03:28:56.888 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:28:56.888 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout disabled 2024-09-14T03:28:56.897 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:28:56.897 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2024-09-14T03:28:56.897 INFO:teuthology.orchestra.run.smithi110.stdout:firewalld.service is not enabled 2024-09-14T03:28:56.897 INFO:teuthology.orchestra.run.smithi110.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-14T03:28:56.897 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mgr to start... 2024-09-14T03:28:56.897 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mgr... 2024-09-14T03:28:57.021 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:56 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:56.997+0000 7f474b5e0ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:28:57.338 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:57.126+0000 7f474b5e0ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:28:57.338 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:57.265+0000 7f474b5e0ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:28:57.594 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:28:57 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3930328553' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:28:57.595 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:57.533+0000 7f474b5e0ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:28:57.595 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:57.593+0000 7f474b5e0ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:28:57.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:57.759+0000 7f474b5e0ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:28:57.974 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsid": "5517b266-7249-11ef-bceb-c7b262605968", 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:28:57.975 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:28:57.976 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:28:57.977 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:28:57.978 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:28:39.370191+0000", 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:28:57.979 INFO:teuthology.orchestra.run.smithi110.stdout:mgr not available, waiting (1/15)... 2024-09-14T03:28:58.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:58.605+0000 7f474b5e0ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:28:58.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:58.669+0000 7f474b5e0ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:28:59.668 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:59.380+0000 7f474b5e0ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:28:59.668 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:59.490+0000 7f474b5e0ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:28:59.668 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:59.607+0000 7f474b5e0ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:28:59.668 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:59.666+0000 7f474b5e0ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:28:59.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:59.790+0000 7f474b5e0ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:28:59.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:59.855+0000 7f474b5e0ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:28:59.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:28:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:28:59.912+0000 7f474b5e0ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:29:00.328 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:00.057+0000 7f474b5e0ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:29:00.328 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:00.121+0000 7f474b5e0ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:29:00.328 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:00.185+0000 7f474b5e0ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:29:00.329 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:00.269+0000 7f474b5e0ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:29:00.329 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:00.326+0000 7f474b5e0ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:29:00.615 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:00 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1094153082' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:29:00.615 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:00.386+0000 7f474b5e0ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:29:01.282 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsid": "5517b266-7249-11ef-bceb-c7b262605968", 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:29:01.283 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:29:01.284 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:29:01.285 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:29:01.285 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:29:01.285 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:29:01.289 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:29:01.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:28:39.370191+0000", 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:29:01.291 INFO:teuthology.orchestra.run.smithi110.stdout:mgr not available, waiting (2/15)... 2024-09-14T03:29:01.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:01.176+0000 7f474b5e0ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:29:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: Activating manager daemon y 2024-09-14T03:29:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: mgrmap e2: y(active, starting, since 0.00323522s) 2024-09-14T03:29:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:29:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:29:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:29:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:29:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:29:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: Manager daemon y is now available 2024-09-14T03:29:01.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:29:01.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:29:01.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' 2024-09-14T03:29:01.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' 2024-09-14T03:29:01.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:01 smithi110 ceph-mon[27365]: from='mgr.14100 172.21.15.110:0/4136914640' entity='mgr.y' 2024-09-14T03:29:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:03 smithi110 ceph-mon[27365]: mgrmap e3: y(active, since 1.00668s) 2024-09-14T03:29:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:04 smithi110 ceph-mon[27365]: mgrmap e4: y(active, since 2s) 2024-09-14T03:29:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:04 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/955257374' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:29:04.460 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsid": "5517b266-7249-11ef-bceb-c7b262605968", 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:29:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:29:04.467 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:29:04.468 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:29:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:28:39.370191+0000", 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:29:04.470 INFO:teuthology.orchestra.run.smithi110.stdout:mgr is available 2024-09-14T03:29:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:05 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4260047330' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-14T03:29:05.852 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:29:05.852 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [global] 2024-09-14T03:29:05.852 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout fsid = 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:29:05.852 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-14T03:29:05.852 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-14T03:29:05.852 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-14T03:29:05.852 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-14T03:29:05.852 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-14T03:29:05.853 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:29:05.853 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-14T03:29:05.853 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-14T03:29:05.853 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2024-09-14T03:29:05.853 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [osd] 2024-09-14T03:29:05.853 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-14T03:29:05.853 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-14T03:29:05.854 INFO:teuthology.orchestra.run.smithi110.stdout:Enabling cephadm module... 2024-09-14T03:29:06.877 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:06 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/181037255' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-14T03:29:06.877 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ignoring --setuser ceph since I am not root 2024-09-14T03:29:06.877 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ignoring --setgroup ceph since I am not root 2024-09-14T03:29:06.878 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:06.629+0000 7f082cbb3ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:29:06.878 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:06.684+0000 7f082cbb3ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:29:06.878 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:06.749+0000 7f082cbb3ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:29:06.878 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:06.875+0000 7f082cbb3ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:29:07.139 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:07 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:07.012+0000 7f082cbb3ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:29:07.396 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:07 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:07.272+0000 7f082cbb3ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:29:07.396 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:07 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:07.332+0000 7f082cbb3ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:29:07.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:07 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/181037255' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-14T03:29:07.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:07 smithi110 ceph-mon[27365]: mgrmap e5: y(active, since 5s) 2024-09-14T03:29:07.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:07 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:07.498+0000 7f082cbb3ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:29:08.411 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:08 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:08.346+0000 7f082cbb3ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:29:08.411 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:08 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:08.409+0000 7f082cbb3ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:29:08.449 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:29:08.449 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-14T03:29:08.450 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:29:08.450 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-14T03:29:08.450 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-14T03:29:08.450 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:29:08.450 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for the mgr to restart... 2024-09-14T03:29:08.450 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mgr epoch 5... 2024-09-14T03:29:08.665 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:08 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4274285121' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-14T03:29:09.405 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.119+0000 7f082cbb3ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:29:09.405 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.228+0000 7f082cbb3ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:29:09.405 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.346+0000 7f082cbb3ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:29:09.405 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.403+0000 7f082cbb3ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:29:09.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.528+0000 7f082cbb3ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:29:09.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.593+0000 7f082cbb3ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:29:09.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.650+0000 7f082cbb3ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:29:10.043 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.782+0000 7f082cbb3ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:29:10.043 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.843+0000 7f082cbb3ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:29:10.043 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.907+0000 7f082cbb3ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:29:10.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:09 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:09.986+0000 7f082cbb3ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:29:10.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:10.041+0000 7f082cbb3ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:29:10.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:10.097+0000 7f082cbb3ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:10.825+0000 7f082cbb3ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: Active manager daemon y restarted 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: Activating manager daemon y 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: osdmap e2: 0 total, 0 up, 0 in 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: mgrmap e6: y(active, starting, since 0.00306956s) 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:29:11.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:29:11.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:29:11.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: Manager daemon y is now available 2024-09-14T03:29:11.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:11.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:11.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:11.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:11.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:29:11.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:10 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:29:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:11 smithi110 ceph-mon[27365]: Found migration_current of "None". Setting to last migration. 2024-09-14T03:29:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:11 smithi110 ceph-mon[27365]: mgrmap e7: y(active, since 1.00504s) 2024-09-14T03:29:12.386 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:12.182+0000 7f07defa2640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:29:12] ENGINE Error in HTTPServer.serve 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: Traceback (most recent call last): 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._connections.run(self.expiration_interval) 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._run(expiration_interval) 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: s = self.context.wrap_socket( 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:29:12.387 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: return self.sslsocket_class._create( 2024-09-14T03:29:12.388 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:29:12.388 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self.do_handshake() 2024-09-14T03:29:12.388 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:29:12.388 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._sslobj.do_handshake() 2024-09-14T03:29:12.388 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:29:12.388 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:12.389 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:29:12.389 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-14T03:29:12.389 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-14T03:29:12.389 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:29:12.389 INFO:teuthology.orchestra.run.smithi110.stdout:mgr epoch 5 is available 2024-09-14T03:29:12.389 INFO:teuthology.orchestra.run.smithi110.stdout:Setting orchestrator backend to cephadm... 2024-09-14T03:29:13.233 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-14T03:29:13.233 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-14T03:29:13.233 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: [14/Sep/2024:03:29:12] ENGINE Bus STARTING 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: [14/Sep/2024:03:29:12] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: [14/Sep/2024:03:29:12] ENGINE Bus STARTED 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: [14/Sep/2024:03:29:12] ENGINE Error in HTTPServer.serve 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:29:13.234 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout: self.do_handshake() 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:13.235 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:13 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:14.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:14 smithi110 ceph-mon[27365]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:14.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:14 smithi110 ceph-mon[27365]: mgrmap e8: y(active, since 2s) 2024-09-14T03:29:14.642 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-14T03:29:14.643 INFO:teuthology.orchestra.run.smithi110.stdout:Generating ssh key... 2024-09-14T03:29:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-mon[27365]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:15.996 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: Generating public/private rsa key pair. 2024-09-14T03:29:15.996 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: Your identification has been saved in /tmp/tmpw4x48wqz/key 2024-09-14T03:29:15.996 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: Your public key has been saved in /tmp/tmpw4x48wqz/key.pub 2024-09-14T03:29:15.996 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: The key fingerprint is: 2024-09-14T03:29:15.996 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: SHA256:pRK9nXtivSl3GQqwICjijsfz/IppL4d7SKaFDPrGu34 ceph-5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:29:15.996 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: The key's randomart image is: 2024-09-14T03:29:15.996 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: +---[RSA 3072]----+ 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: | | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: | . | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: | . . . . | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: |+ . . ...= . | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: |*o ...Soo | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: |oo+ .. .o . | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: |oO o +.o. o | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: |o.%+E ..oooo | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: | =B%=o. oo. | 2024-09-14T03:29:15.997 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: +----[SHA256]-----+ 2024-09-14T03:29:16.280 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:16 smithi110 ceph-mon[27365]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:16.280 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:16 smithi110 ceph-mon[27365]: Generating ssh key... 2024-09-14T03:29:16.280 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:16 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:16.280 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:16 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:17.464 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClEyluXT/LARojX2PFYJ9c5S14eGgInxA0Y0zVNFr2btLtdjYo7QD08Xk0XAVjwHK2KjH+Ol3YLwCgzOUJCF2Usnh9lEE2RB0tD8+5UMLYHgY3wQH9PN05Vh/xvA6fQKWRPQ3dAuq976Q+Yq0IZZDzkhRM/dcbyjkSDzpCLxYR3oqgjjkF5+51HbaRAOmGq/sVTEaIVBhpGOUkdAZl9+DHY5Xvg3eppYwmJx2vm8s8poMEr5cNUQCyriqJjCSZo9y9vIZpWx963H0IBAPM4FlL9wJs+dlNzivp19Cvytd+DrjK+7x56R4RHaL6cU9bGUq+b2gPGzLUA1D6IIg/iRaECs+X71/6uAFGoS1hy2aTaACEmeTHx18bK9L+2IVUlCT71XQvWMMC7HlSz3rXLjmHZSDNg+4lpFL02OtdFnxA7sb7QznsyTZFkQOFltDzghfLQM8ZXR7ljpmYNz/IwnOZLH1x0jCDakZsB7QC7h2YFlY6hzFNefWS2W52GjMtvRU= ceph-5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:29:17.464 INFO:teuthology.orchestra.run.smithi110.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-14T03:29:17.464 INFO:teuthology.orchestra.run.smithi110.stdout:Adding key to root@localhost authorized_keys... 2024-09-14T03:29:17.474 INFO:teuthology.orchestra.run.smithi110.stdout:Adding host smithi110... 2024-09-14T03:29:17.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:17 smithi110 ceph-mon[27365]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:18.823 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:18 smithi110 ceph-mon[27365]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi110", "addr": "172.21.15.110", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:19.758 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:19 smithi110 ceph-mon[27365]: Deploying cephadm binary to smithi110 2024-09-14T03:29:21.389 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout Added host 'smithi110' with addr '172.21.15.110' 2024-09-14T03:29:21.389 INFO:teuthology.orchestra.run.smithi110.stdout:Deploying unmanaged mon service... 2024-09-14T03:29:21.932 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:21 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:21.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:21 smithi110 ceph-mon[27365]: Added host smithi110 2024-09-14T03:29:21.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:21 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:22.750 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-14T03:29:22.750 INFO:teuthology.orchestra.run.smithi110.stdout:Deploying unmanaged mgr service... 2024-09-14T03:29:23.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:23 smithi110 ceph-mon[27365]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:23.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:23 smithi110 ceph-mon[27365]: Saving service mon spec with placement count:5 2024-09-14T03:29:23.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:23 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:24.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:24 smithi110 ceph-mon[27365]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:24.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:24 smithi110 ceph-mon[27365]: Saving service mgr spec with placement count:2 2024-09-14T03:29:24.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:24 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:24.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:24 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:24.451 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-14T03:29:26.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:25 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:26.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:25 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1036434223' entity='client.admin' 2024-09-14T03:29:26.946 INFO:teuthology.orchestra.run.smithi110.stdout:Enabling the dashboard module... 2024-09-14T03:29:27.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:27 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/478825865' entity='client.admin' 2024-09-14T03:29:28.549 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2883511379' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-14T03:29:28.549 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-mon[27365]: from='mgr.14118 172.21.15.110:0/901298662' entity='mgr.y' 2024-09-14T03:29:28.549 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ignoring --setuser ceph since I am not root 2024-09-14T03:29:28.550 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ignoring --setgroup ceph since I am not root 2024-09-14T03:29:28.550 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:28.492+0000 7f48440e4ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:29:28.550 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:28.546+0000 7f48440e4ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:29:28.875 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:28.614+0000 7f48440e4ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:29:28.875 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:28.738+0000 7f48440e4ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:29:28.876 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:28.873+0000 7f48440e4ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:29:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:29 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2883511379' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-14T03:29:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:29 smithi110 ceph-mon[27365]: mgrmap e9: y(active, since 17s) 2024-09-14T03:29:29.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:29 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:29.133+0000 7f48440e4ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:29:29.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:29 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:29.193+0000 7f48440e4ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:29:29.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:29 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:29.359+0000 7f48440e4ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:29:30.288 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:30.220+0000 7f48440e4ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:29:30.288 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:30.286+0000 7f48440e4ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:29:30.342 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:29:30.343 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-14T03:29:30.343 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:29:30.343 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-14T03:29:30.343 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-14T03:29:30.343 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:29:30.343 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for the mgr to restart... 2024-09-14T03:29:30.343 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mgr epoch 9... 2024-09-14T03:29:30.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:30 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3625209164' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-14T03:29:31.291 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.004+0000 7f48440e4ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:29:31.291 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.113+0000 7f48440e4ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:29:31.291 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.231+0000 7f48440e4ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:29:31.291 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.289+0000 7f48440e4ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:29:31.569 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.412+0000 7f48440e4ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:29:31.569 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.477+0000 7f48440e4ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:29:31.569 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.533+0000 7f48440e4ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:29:31.821 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.673+0000 7f48440e4ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:29:31.821 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.739+0000 7f48440e4ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:29:31.821 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.807+0000 7f48440e4ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:29:32.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.891+0000 7f48440e4ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:29:32.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:31.950+0000 7f48440e4ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:29:32.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:32.011+0000 7f48440e4ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:29:33.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:32.775+0000 7f48440e4ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:29:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: Active manager daemon y restarted 2024-09-14T03:29:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: Activating manager daemon y 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: osdmap e3: 0 total, 0 up, 0 in 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: mgrmap e10: y(active, starting, since 0.00351935s) 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: Manager daemon y is now available 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:29:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:34.246+0000 7f47f4c42640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:29:34] ENGINE Error in HTTPServer.serve 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: Traceback (most recent call last): 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._connections.run(self.expiration_interval) 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._run(expiration_interval) 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:29:34.517 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: s = self.context.wrap_socket( 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: return self.sslsocket_class._create( 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self.do_handshake() 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._sslobj.do_handshake() 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:29:34.518 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:29:34.519 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:29:34.519 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-14T03:29:34.519 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-14T03:29:34.519 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:29:34.519 INFO:teuthology.orchestra.run.smithi110.stdout:mgr epoch 9 is available 2024-09-14T03:29:34.520 INFO:teuthology.orchestra.run.smithi110.stdout:Generating a dashboard self-signed certificate... 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-mon[27365]: mgrmap e11: y(active, since 1.00559s) 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-mon[27365]: [14/Sep/2024:03:29:34] ENGINE Bus STARTING 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-mon[27365]: [14/Sep/2024:03:29:34] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-mon[27365]: [14/Sep/2024:03:29:34] ENGINE Bus STARTED 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-mon[27365]: [14/Sep/2024:03:29:34] ENGINE Error in HTTPServer.serve 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:29:35.131 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:29:35.132 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:29:35.133 INFO:journalctl@ceph.mon.a.smithi110.stdout: self.do_handshake() 2024-09-14T03:29:35.133 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:29:35.133 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:29:35.133 INFO:journalctl@ceph.mon.a.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:29:35.133 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:35.742 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-14T03:29:35.742 INFO:teuthology.orchestra.run.smithi110.stdout:Creating initial admin user... 2024-09-14T03:29:36.285 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:36 smithi110 ceph-mon[27365]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:36.285 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:36 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:36.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:36 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:36.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:36 smithi110 ceph-mon[27365]: mgrmap e12: y(active, since 2s) 2024-09-14T03:29:36.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:36 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:37.321 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$T9KKq5xYiLPYTabWgncRoO04EfumJI344WcTJFP2h/kkpzE6fvQwK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726284576, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-14T03:29:37.321 INFO:teuthology.orchestra.run.smithi110.stdout:Fetching dashboard port number... 2024-09-14T03:29:37.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:37 smithi110 ceph-mon[27365]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:37.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:38.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:38 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4057270342' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-14T03:29:38.880 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 8443 2024-09-14T03:29:38.889 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:29:38.889 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout disabled 2024-09-14T03:29:38.897 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:29:38.897 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2024-09-14T03:29:38.897 INFO:teuthology.orchestra.run.smithi110.stdout:firewalld.service is not enabled 2024-09-14T03:29:38.897 INFO:teuthology.orchestra.run.smithi110.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-14T03:29:38.899 INFO:teuthology.orchestra.run.smithi110.stdout:Ceph Dashboard is now available at: 2024-09-14T03:29:38.899 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:38.899 INFO:teuthology.orchestra.run.smithi110.stdout: URL: https://smithi110.front.sepia.ceph.com:8443/ 2024-09-14T03:29:38.899 INFO:teuthology.orchestra.run.smithi110.stdout: User: admin 2024-09-14T03:29:38.899 INFO:teuthology.orchestra.run.smithi110.stdout: Password: zqe6j2ih6f 2024-09-14T03:29:38.899 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:38.900 INFO:teuthology.orchestra.run.smithi110.stdout:Saving cluster configuration to /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config directory 2024-09-14T03:29:38.900 INFO:teuthology.orchestra.run.smithi110.stdout:Enabling autotune for osd_memory_target 2024-09-14T03:29:40.606 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:40 smithi110 ceph-mon[27365]: mgrmap e13: y(active, since 6s) 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout:Or, if you are only running a single cluster on this host: 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:41.134 INFO:teuthology.orchestra.run.smithi110.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-14T03:29:41.135 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:41.135 INFO:teuthology.orchestra.run.smithi110.stdout: ceph telemetry on 2024-09-14T03:29:41.135 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:41.135 INFO:teuthology.orchestra.run.smithi110.stdout:For more information see: 2024-09-14T03:29:41.135 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:41.135 INFO:teuthology.orchestra.run.smithi110.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-14T03:29:41.135 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:41.135 INFO:teuthology.orchestra.run.smithi110.stdout:Bootstrap complete. 2024-09-14T03:29:41.168 INFO:tasks.cephadm:Fetching config... 2024-09-14T03:29:41.168 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:29:41.169 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-14T03:29:41.184 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-14T03:29:41.185 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:29:41.185 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-14T03:29:41.240 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-14T03:29:41.240 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:29:41.240 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-14T03:29:41.305 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-14T03:29:41.305 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:29:41.305 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-14T03:29:41.360 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-14T03:29:41.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClEyluXT/LARojX2PFYJ9c5S14eGgInxA0Y0zVNFr2btLtdjYo7QD08Xk0XAVjwHK2KjH+Ol3YLwCgzOUJCF2Usnh9lEE2RB0tD8+5UMLYHgY3wQH9PN05Vh/xvA6fQKWRPQ3dAuq976Q+Yq0IZZDzkhRM/dcbyjkSDzpCLxYR3oqgjjkF5+51HbaRAOmGq/sVTEaIVBhpGOUkdAZl9+DHY5Xvg3eppYwmJx2vm8s8poMEr5cNUQCyriqJjCSZo9y9vIZpWx963H0IBAPM4FlL9wJs+dlNzivp19Cvytd+DrjK+7x56R4RHaL6cU9bGUq+b2gPGzLUA1D6IIg/iRaECs+X71/6uAFGoS1hy2aTaACEmeTHx18bK9L+2IVUlCT71XQvWMMC7HlSz3rXLjmHZSDNg+4lpFL02OtdFnxA7sb7QznsyTZFkQOFltDzghfLQM8ZXR7ljpmYNz/IwnOZLH1x0jCDakZsB7QC7h2YFlY6hzFNefWS2W52GjMtvRU= ceph-5517b266-7249-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-14T03:29:41.434 INFO:teuthology.orchestra.run.smithi110.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClEyluXT/LARojX2PFYJ9c5S14eGgInxA0Y0zVNFr2btLtdjYo7QD08Xk0XAVjwHK2KjH+Ol3YLwCgzOUJCF2Usnh9lEE2RB0tD8+5UMLYHgY3wQH9PN05Vh/xvA6fQKWRPQ3dAuq976Q+Yq0IZZDzkhRM/dcbyjkSDzpCLxYR3oqgjjkF5+51HbaRAOmGq/sVTEaIVBhpGOUkdAZl9+DHY5Xvg3eppYwmJx2vm8s8poMEr5cNUQCyriqJjCSZo9y9vIZpWx963H0IBAPM4FlL9wJs+dlNzivp19Cvytd+DrjK+7x56R4RHaL6cU9bGUq+b2gPGzLUA1D6IIg/iRaECs+X71/6uAFGoS1hy2aTaACEmeTHx18bK9L+2IVUlCT71XQvWMMC7HlSz3rXLjmHZSDNg+4lpFL02OtdFnxA7sb7QznsyTZFkQOFltDzghfLQM8ZXR7ljpmYNz/IwnOZLH1x0jCDakZsB7QC7h2YFlY6hzFNefWS2W52GjMtvRU= ceph-5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:29:41.445 DEBUG:teuthology.orchestra.run.smithi195:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClEyluXT/LARojX2PFYJ9c5S14eGgInxA0Y0zVNFr2btLtdjYo7QD08Xk0XAVjwHK2KjH+Ol3YLwCgzOUJCF2Usnh9lEE2RB0tD8+5UMLYHgY3wQH9PN05Vh/xvA6fQKWRPQ3dAuq976Q+Yq0IZZDzkhRM/dcbyjkSDzpCLxYR3oqgjjkF5+51HbaRAOmGq/sVTEaIVBhpGOUkdAZl9+DHY5Xvg3eppYwmJx2vm8s8poMEr5cNUQCyriqJjCSZo9y9vIZpWx963H0IBAPM4FlL9wJs+dlNzivp19Cvytd+DrjK+7x56R4RHaL6cU9bGUq+b2gPGzLUA1D6IIg/iRaECs+X71/6uAFGoS1hy2aTaACEmeTHx18bK9L+2IVUlCT71XQvWMMC7HlSz3rXLjmHZSDNg+4lpFL02OtdFnxA7sb7QznsyTZFkQOFltDzghfLQM8ZXR7ljpmYNz/IwnOZLH1x0jCDakZsB7QC7h2YFlY6hzFNefWS2W52GjMtvRU= ceph-5517b266-7249-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-14T03:29:41.564 INFO:teuthology.orchestra.run.smithi195.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClEyluXT/LARojX2PFYJ9c5S14eGgInxA0Y0zVNFr2btLtdjYo7QD08Xk0XAVjwHK2KjH+Ol3YLwCgzOUJCF2Usnh9lEE2RB0tD8+5UMLYHgY3wQH9PN05Vh/xvA6fQKWRPQ3dAuq976Q+Yq0IZZDzkhRM/dcbyjkSDzpCLxYR3oqgjjkF5+51HbaRAOmGq/sVTEaIVBhpGOUkdAZl9+DHY5Xvg3eppYwmJx2vm8s8poMEr5cNUQCyriqJjCSZo9y9vIZpWx963H0IBAPM4FlL9wJs+dlNzivp19Cvytd+DrjK+7x56R4RHaL6cU9bGUq+b2gPGzLUA1D6IIg/iRaECs+X71/6uAFGoS1hy2aTaACEmeTHx18bK9L+2IVUlCT71XQvWMMC7HlSz3rXLjmHZSDNg+4lpFL02OtdFnxA7sb7QznsyTZFkQOFltDzghfLQM8ZXR7ljpmYNz/IwnOZLH1x0jCDakZsB7QC7h2YFlY6hzFNefWS2W52GjMtvRU= ceph-5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:29:41.577 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-14T03:29:41.823 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:29:41.944 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:41 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1939579022' entity='client.admin' 2024-09-14T03:29:44.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:44.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:44.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi110", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:29:44.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:44.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:44.336 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:44.661 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-14T03:29:44.661 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-14T03:29:44.897 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:29:45.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:45 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1984452525' entity='client.admin' 2024-09-14T03:29:45.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:45 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:45.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:45 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:47.269 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi195 2024-09-14T03:29:47.269 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:29:47.270 DEBUG:teuthology.orchestra.run.smithi195:> dd of=/etc/ceph/ceph.conf 2024-09-14T03:29:47.290 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:29:47.290 DEBUG:teuthology.orchestra.run.smithi195:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:29:47.348 INFO:tasks.cephadm:Adding host smithi195 to orchestrator... 2024-09-14T03:29:47.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch host add smithi195 2024-09-14T03:29:47.630 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:29:47.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:47.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:47.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:47.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:29:47.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:29:47.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:29:47.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:29:47.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:29:47.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:29:47.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:47.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:47.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:47 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:49.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:49 smithi110 ceph-mon[27365]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi195", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:29:50.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:50 smithi110 ceph-mon[27365]: Deploying cephadm binary to smithi195 2024-09-14T03:29:51.936 INFO:teuthology.orchestra.run.smithi110.stdout:Added host 'smithi195' with addr '172.21.15.195' 2024-09-14T03:29:52.539 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-14T03:29:52.775 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:29:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:52 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:52 smithi110 ceph-mon[27365]: Added host smithi195 2024-09-14T03:29:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:52 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:29:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:52 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:52 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:54.430 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:29:54.430 INFO:teuthology.orchestra.run.smithi110.stdout:[{"addr": "172.21.15.110", "hostname": "smithi110", "labels": [], "status": ""}, {"addr": "172.21.15.195", "hostname": "smithi195", "labels": [], "status": ""}] 2024-09-14T03:29:54.430 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:54 smithi110 ceph-mon[27365]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:29:54.430 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:54 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:54.985 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-14T03:29:54.986 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-14T03:29:55.221 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi195", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:29:56.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:29:57.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:57 smithi110 ceph-mon[27365]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:29:57.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:57 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:29:57.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:57 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:57.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:57 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:57.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:57 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:29:57.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:57 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2193779199' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-14T03:29:57.895 INFO:teuthology.orchestra.run.smithi110.stderr:adjusted tunables profile to default 2024-09-14T03:29:58.459 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:58 smithi110 ceph-mon[27365]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:29:58.459 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:58 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:29:58.459 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:58 smithi110 ceph-mon[27365]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:29:58.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:58 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2193779199' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-14T03:29:58.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:29:58 smithi110 ceph-mon[27365]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:29:58.487 INFO:tasks.cephadm:Adding mon.a on smithi110 2024-09-14T03:29:58.487 INFO:tasks.cephadm:Adding mon.c on smithi110 2024-09-14T03:29:58.488 INFO:tasks.cephadm:Adding mon.b on smithi195 2024-09-14T03:29:58.488 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi195:172.21.15.195=b' 2024-09-14T03:29:58.741 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:30:00.313 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled mon update... 2024-09-14T03:30:00.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:00 smithi110 ceph-mon[27365]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:00.892 DEBUG:teuthology.orchestra.run.smithi110:mon.c> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.c.service 2024-09-14T03:30:00.895 DEBUG:teuthology.orchestra.run.smithi195:mon.b> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.b.service 2024-09-14T03:30:00.897 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:30:00.898 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:30:01.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi195:172.21.15.195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:01.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: Saving service mon spec with placement smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi195:172.21.15.195=b;count:3 2024-09-14T03:30:01.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:01.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:01.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:01.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:01.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:01.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:01.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:01.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:01 smithi110 ceph-mon[27365]: Deploying daemon mon.b on smithi195 2024-09-14T03:30:01.945 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:30:02.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:02 smithi110 ceph-mon[27365]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:04.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:04 smithi110 ceph-mon[27365]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:04.887 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:30:04.887 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","modified":"2024-09-14T03:28:36.548032Z","created":"2024-09-14T03:28:36.548032Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-14T03:30:04.887 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-14T03:30:05.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:05 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/310897107' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:30:06.874 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:30:06.874 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:30:07.139 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:30:10.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:30:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: mon.a calling monitor election 2024-09-14T03:30:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: mon.b calling monitor election 2024-09-14T03:30:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:30:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: monmap e2: 2 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:30:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: fsmap 2024-09-14T03:30:10.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:30:10.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: mgrmap e13: y(active, since 37s) 2024-09-14T03:30:10.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: overall HEALTH_OK 2024-09-14T03:30:10.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:10.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:10.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:10.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:10.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:10.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:10 smithi110 ceph-mon[27365]: Deploying daemon mon.c on smithi110 2024-09-14T03:30:10.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:10 smithi195 ceph-mon[30695]: Deploying daemon mon.c on smithi110 2024-09-14T03:30:11.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:11 smithi110 ceph-mon[27365]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:11.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:11 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:11 smithi195 ceph-mon[30695]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:11 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:11.796 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:30:11.797 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":2,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","modified":"2024-09-14T03:30:05.375518Z","created":"2024-09-14T03:28:36.548032Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:3300","nonce":0},{"type":"v1","addr":"172.21.15.195:6789","nonce":0}]},"addr":"172.21.15.195:6789/0","public_addr":"172.21.15.195:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-14T03:30:11.797 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 2 2024-09-14T03:30:12.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:12 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/2898822597' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:30:12.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:12 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/2898822597' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:30:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 systemd[1]: Starting Ceph mon.c for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:30:13.375 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:30:13.376 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:30:13.501 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[27365]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.501 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 podman[35348]: 2024-09-14 03:30:13.149888299 +0000 UTC m=+0.016682059 image pull ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6 2024-09-14T03:30:13.501 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 podman[35348]: 2024-09-14 03:30:13.289838204 +0000 UTC m=+0.156631964 container create 8e2894672cd8a7acaae1403b8d1c9bb67e4bdd64f1313d6f1c35cdc31e7650c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:30:13.625 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:30:13.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:13 smithi195 ceph-mon[30695]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 podman[35348]: 2024-09-14 03:30:13.525603069 +0000 UTC m=+0.392396832 container init 8e2894672cd8a7acaae1403b8d1c9bb67e4bdd64f1313d6f1c35cdc31e7650c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 podman[35348]: 2024-09-14 03:30:13.529388884 +0000 UTC m=+0.396182645 container start 8e2894672cd8a7acaae1403b8d1c9bb67e4bdd64f1313d6f1c35cdc31e7650c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable), process ceph-mon, pid 2 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: pidfile_write: ignore empty --pid-file 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: load: jerasure load: lrc 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: RocksDB version: 6.15.5 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Compile date Sep 13 2024 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: DB SUMMARY 2024-09-14T03:30:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: DB Session ID: 443U30TT929NTJ7PGRA1 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.env: 0x5611c776e440 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.fs: Posix File System 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.info_log: 0x5611c806cec0 2024-09-14T03:30:13.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:30:13.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.statistics: (nil) 2024-09-14T03:30:13.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.use_fsync: 0 2024-09-14T03:30:13.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:30:13.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.db_log_dir: 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.write_buffer_manager: 0x5611c812a7b0 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:30:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.unordered_write: 0 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:30:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.row_cache: None 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.wal_filter: None 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.preserve_deletes: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:30:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:30:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_open_files: -1 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:30:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Compression algorithms supported: 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: kZSTD supported: 0 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:30:13.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.merge_operator: 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_filter: None 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5611c8013a38) 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_type: 0 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_block_index_type: 0 2024-09-14T03:30:13.957 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_shortening: 1 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: hash_index_allow_collision: 1 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: checksum: 1 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: no_block_cache: 0 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache: 0x5611c80491f0 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_options: 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: capacity : 536870912 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: num_shard_bits : 4 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: strict_capacity_limit : 0 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:30:13.958 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_compressed: (nil) 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: persistent_cache: (nil) 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_size: 4096 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_size_deviation: 10 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_restart_interval: 16 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_block_restart_interval: 1 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: metadata_block_size: 4096 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: partition_filters: 0 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: use_delta_encoding: 1 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: whole_key_filtering: 1 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: verify_compression: 0 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:30:13.959 INFO:journalctl@ceph.mon.c.smithi110.stdout: format_version: 4 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout: enable_index_compression: 1 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_align: 0 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compression: NoCompression 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.num_levels: 7 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:30:13.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:30:13.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:30:13.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:30:13.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:30:13.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:30:13.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:30:13.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:30:13.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.arena_block_size: 4194304 2024-09-14T03:30:13.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:30:13.964 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:30:13.964 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-14T03:30:13.964 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:30:13.964 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:30:13.964 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:30:13.965 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.ttl: 2592000 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:30:13.966 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284613554052, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284613554465, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726284613, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "507c9d8f-abbd-4524-ac6f-48c48f5087d4", "db_session_id": "443U30TT929NTJ7PGRA1"}} 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284613554905, "job": 1, "event": "recovery_finished"} 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5611c807c700 2024-09-14T03:30:13.967 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: DB pointer 0x5611c804c000 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** DB Stats ** 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:30:13.968 INFO:journalctl@ceph.mon.c.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:30:13.969 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative compaction: 0.00 GB write, 0.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval compaction: 0.00 GB write, 0.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:30:13.970 INFO:journalctl@ceph.mon.c.smithi110.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:30:13.971 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative compaction: 0.00 GB write, 0.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: using public_addrv [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: starting mon.c rank -1 at public addrs [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] at bind addrs [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(???) e0 preinit fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).mds e1 new map 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout: e1 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:30:13.972 INFO:journalctl@ceph.mon.c.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout: legacy client fscid: -1 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout: No filesystems configured 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:30:13.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi195:172.21.15.195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: Saving service mon spec with placement smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi195:172.21.15.195=b;count:3 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: Deploying daemon mon.b on smithi195 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/310897107' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.a calling monitor election 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.b calling monitor election 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:13.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: monmap e2: 2 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: fsmap 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mgrmap e13: y(active, since 37s) 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: overall HEALTH_OK 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: Deploying daemon mon.c on smithi110 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.976 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:13.977 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/2898822597' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:30:13.977 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:13.977 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 ceph-mon[35374]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-14T03:30:13.977 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 bash[35348]: 8e2894672cd8a7acaae1403b8d1c9bb67e4bdd64f1313d6f1c35cdc31e7650c4 2024-09-14T03:30:13.977 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:13 smithi110 systemd[1]: Started Ceph mon.c for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:30:18.834 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:30:18.835 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":3,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","modified":"2024-09-14T03:30:13.577071Z","created":"2024-09-14T03:28:36.548032Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:3300","nonce":0},{"type":"v1","addr":"172.21.15.195:6789","nonce":0}]},"addr":"172.21.15.195:6789/0","public_addr":"172.21.15.195:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3301","nonce":0},{"type":"v1","addr":"172.21.15.110:6790","nonce":0}]},"addr":"172.21.15.110:6790/0","public_addr":"172.21.15.110:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-14T03:30:18.835 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 3 2024-09-14T03:30:18.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:30:18.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: mon.a calling monitor election 2024-09-14T03:30:18.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: mon.b calling monitor election 2024-09-14T03:30:18.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: fsmap 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:30:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: mgrmap e13: y(active, since 45s) 2024-09-14T03:30:18.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-14T03:30:18.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-14T03:30:18.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-14T03:30:18.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: mon.c (rank 2) addr [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] is down (out of quorum) 2024-09-14T03:30:18.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:18.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:18.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:18.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:30:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: mon.a calling monitor election 2024-09-14T03:30:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: mon.b calling monitor election 2024-09-14T03:30:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: fsmap 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: mgrmap e13: y(active, since 45s) 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-14T03:30:18.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: mon.c (rank 2) addr [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] is down (out of quorum) 2024-09-14T03:30:18.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:18.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:18.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:18.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:19.445 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-14T03:30:19.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-14T03:30:19.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:19.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:19 smithi195 ceph-mon[30695]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:19.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:19 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/795102255' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:30:19.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:19.819 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:30:19.940 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:19.940 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:19 smithi110 ceph-mon[27365]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:19.940 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:19 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/795102255' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:30:19.940 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:20.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:30:20.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: mon.a calling monitor election 2024-09-14T03:30:20.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: mon.b calling monitor election 2024-09-14T03:30:20.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:30:20.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:20.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:20.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:20.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:20.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:20.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:20.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:20.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:20.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:30:20.917 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:30:20.917 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: fsmap 2024-09-14T03:30:20.917 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:30:20.917 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: mgrmap e13: y(active, since 45s) 2024-09-14T03:30:20.917 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-14T03:30:20.917 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-14T03:30:20.918 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-14T03:30:20.918 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: mon.c (rank 2) addr [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] is down (out of quorum) 2024-09-14T03:30:20.918 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:20.918 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:20.918 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:20.918 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:20.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:20.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:20.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/795102255' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:30:20.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:20 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: mon.c calling monitor election 2024-09-14T03:30:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: mon.c calling monitor election 2024-09-14T03:30:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: mon.a calling monitor election 2024-09-14T03:30:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: mon.b calling monitor election 2024-09-14T03:30:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: fsmap 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: mgrmap e13: y(active, since 47s) 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: Cluster is now healthy 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: overall HEALTH_OK 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:21.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: mon.c calling monitor election 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: mon.c calling monitor election 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: mon.a calling monitor election 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: mon.b calling monitor election 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: fsmap 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: mgrmap e13: y(active, since 47s) 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: Cluster is now healthy 2024-09-14T03:30:21.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: overall HEALTH_OK 2024-09-14T03:30:21.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:21.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:21.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:21.668 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: mon.c calling monitor election 2024-09-14T03:30:21.668 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: mon.c calling monitor election 2024-09-14T03:30:21.668 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: mon.a calling monitor election 2024-09-14T03:30:21.668 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: mon.b calling monitor election 2024-09-14T03:30:21.668 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:30:21.668 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:30:21.668 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: fsmap 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: mgrmap e13: y(active, since 47s) 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: Cluster is now healthy 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: overall HEALTH_OK 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:21.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:22.145 INFO:teuthology.orchestra.run.smithi110.stdout:# minimal ceph.conf for 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:30:22.145 INFO:teuthology.orchestra.run.smithi110.stdout:[global] 2024-09-14T03:30:22.145 INFO:teuthology.orchestra.run.smithi110.stdout: fsid = 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:30:22.145 INFO:teuthology.orchestra.run.smithi110.stdout: mon_host = [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] 2024-09-14T03:30:22.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:30:22.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:30:22.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[35374]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:30:22.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[35374]: Reconfiguring daemon mon.a on smithi110 2024-09-14T03:30:22.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[35374]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:22.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:22.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3299107183' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:22.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:30:22.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:30:22.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[27365]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:30:22.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[27365]: Reconfiguring daemon mon.a on smithi110 2024-09-14T03:30:22.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[27365]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:22.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:22.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:22 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3299107183' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:22 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:30:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:22 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:30:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:22 smithi195 ceph-mon[30695]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:30:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:22 smithi195 ceph-mon[30695]: Reconfiguring daemon mon.a on smithi110 2024-09-14T03:30:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:22 smithi195 ceph-mon[30695]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:22 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:30:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:22 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3299107183' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:23.041 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-14T03:30:23.041 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:30:23.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.conf 2024-09-14T03:30:23.068 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:30:23.068 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:30:23.137 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:30:23.137 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.conf 2024-09-14T03:30:23.170 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:30:23.170 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:30:23.238 INFO:tasks.cephadm:Adding mgr.y on smithi110 2024-09-14T03:30:23.238 INFO:tasks.cephadm:Adding mgr.x on smithi195 2024-09-14T03:30:23.239 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi110=y;smithi195=x' 2024-09-14T03:30:23.521 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:30:23.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:23.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:23.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:30:23.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: Reconfiguring daemon mon.c on smithi110 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: Reconfiguring daemon mon.c on smithi110 2024-09-14T03:30:23.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:23.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:23 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: Reconfiguring daemon mon.c on smithi110 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:23 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:25.364 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:25.364 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:25.364 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:25 smithi195 ceph-mon[30695]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:30:25.364 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:25.364 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:25.364 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:25.364 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:25 smithi195 ceph-mon[30695]: Reconfiguring daemon mon.b on smithi195 2024-09-14T03:30:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[27365]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[27365]: Reconfiguring daemon mon.b on smithi195 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[35374]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:25 smithi110 ceph-mon[35374]: Reconfiguring daemon mon.b on smithi195 2024-09-14T03:30:25.621 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled mgr update... 2024-09-14T03:30:26.217 DEBUG:teuthology.orchestra.run.smithi195:mgr.x> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x.service 2024-09-14T03:30:26.219 DEBUG:tasks.cephadm:set 0 configs 2024-09-14T03:30:26.220 INFO:tasks.cephadm:Deploying OSDs... 2024-09-14T03:30:26.220 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:30:26.220 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:30:26.240 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:30:26.240 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_1 2024-09-14T03:30:26.299 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:30:26.299 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:30:26.299 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 5h/5d Inode: 939 Links: 1 2024-09-14T03:30:26.300 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:30:26.300 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:30:26.300 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2024-09-14 03:30:12.977145388 +0000 2024-09-14T03:30:26.300 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2024-09-14 03:26:11.295214236 +0000 2024-09-14T03:30:26.300 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2024-09-14 03:26:11.295214236 +0000 2024-09-14T03:30:26.300 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: 2024-09-14 03:26:11.295214236 +0000 2024-09-14T03:30:26.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:30:26.369 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2024-09-14T03:30:26.370 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2024-09-14T03:30:26.370 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000123572 s, 4.1 MB/s 2024-09-14T03:30:26.371 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:30:26.430 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_2 2024-09-14T03:30:26.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:26 smithi195 ceph-mon[30695]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:26.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:26 smithi195 ceph-mon[30695]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi110=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:26.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:26 smithi195 ceph-mon[30695]: Saving service mgr spec with placement smithi110=y;smithi195=x;count:2 2024-09-14T03:30:26.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:26 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:26.488 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:30:26.488 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:30:26.488 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 5h/5d Inode: 940 Links: 1 2024-09-14T03:30:26.488 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:30:26.488 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:30:26.488 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2024-09-14 03:30:12.977145388 +0000 2024-09-14T03:30:26.489 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2024-09-14 03:26:11.295214236 +0000 2024-09-14T03:30:26.489 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2024-09-14 03:26:11.295214236 +0000 2024-09-14T03:30:26.489 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: 2024-09-14 03:26:11.295214236 +0000 2024-09-14T03:30:26.489 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:30:26.554 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2024-09-14T03:30:26.554 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2024-09-14T03:30:26.554 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000107481 s, 4.8 MB/s 2024-09-14T03:30:26.555 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:30:26.613 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_3 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 5h/5d Inode: 955 Links: 1 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2024-09-14 03:30:12.978145372 +0000 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2024-09-14 03:26:11.296214219 +0000 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2024-09-14 03:26:11.296214219 +0000 2024-09-14T03:30:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: 2024-09-14 03:26:11.296214219 +0000 2024-09-14T03:30:26.672 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:30:26.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:26 smithi110 ceph-mon[27365]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:26.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:26 smithi110 ceph-mon[27365]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi110=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:26.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:26 smithi110 ceph-mon[27365]: Saving service mgr spec with placement smithi110=y;smithi195=x;count:2 2024-09-14T03:30:26.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:26 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:26.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:26 smithi110 ceph-mon[35374]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:26.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:26 smithi110 ceph-mon[35374]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi110=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:26.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:26 smithi110 ceph-mon[35374]: Saving service mgr spec with placement smithi110=y;smithi195=x;count:2 2024-09-14T03:30:26.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:26 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:26.721 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2024-09-14T03:30:26.722 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2024-09-14T03:30:26.722 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000118178 s, 4.3 MB/s 2024-09-14T03:30:26.724 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:30:26.781 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_4 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 5h/5d Inode: 973 Links: 1 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2024-09-14 03:30:12.978145372 +0000 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2024-09-14 03:26:11.298214185 +0000 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2024-09-14 03:26:11.298214185 +0000 2024-09-14T03:30:26.838 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: 2024-09-14 03:26:11.298214185 +0000 2024-09-14T03:30:26.838 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:30:26.901 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2024-09-14T03:30:26.901 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2024-09-14T03:30:26.901 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000108536 s, 4.7 MB/s 2024-09-14T03:30:26.903 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:30:26.960 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:30:26.960 DEBUG:teuthology.orchestra.run.smithi195:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:30:26.976 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:30:26.976 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_1 2024-09-14T03:30:27.032 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:30:27.032 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:30:27.032 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 960 Links: 1 2024-09-14T03:30:27.032 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:30:27.032 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:30:27.032 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-09-14 03:30:04.364732605 +0000 2024-09-14T03:30:27.032 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-09-14 03:26:08.790131895 +0000 2024-09-14T03:30:27.032 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-09-14 03:26:08.790131895 +0000 2024-09-14T03:30:27.033 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-09-14 03:26:08.790131895 +0000 2024-09-14T03:30:27.033 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:30:27.096 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-09-14T03:30:27.096 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-09-14T03:30:27.096 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000113977 s, 4.5 MB/s 2024-09-14T03:30:27.097 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:30:27.154 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_2 2024-09-14T03:30:27.209 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:30:27.209 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:30:27.210 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 969 Links: 1 2024-09-14T03:30:27.210 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:30:27.210 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:30:27.210 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-09-14 03:30:04.364732605 +0000 2024-09-14T03:30:27.210 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-09-14 03:26:08.790131895 +0000 2024-09-14T03:30:27.210 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-09-14 03:26:08.790131895 +0000 2024-09-14T03:30:27.210 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-09-14 03:26:08.790131895 +0000 2024-09-14T03:30:27.210 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:30:27.273 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-09-14T03:30:27.274 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-09-14T03:30:27.274 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000108918 s, 4.7 MB/s 2024-09-14T03:30:27.275 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:30:27.333 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_3 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 983 Links: 1 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-09-14 03:30:04.364732605 +0000 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-09-14 03:26:08.791131953 +0000 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-09-14 03:26:08.791131953 +0000 2024-09-14T03:30:27.388 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-09-14 03:26:08.791131953 +0000 2024-09-14T03:30:27.389 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:30:27.451 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-09-14T03:30:27.451 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-09-14T03:30:27.451 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000111259 s, 4.6 MB/s 2024-09-14T03:30:27.452 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:30:27.509 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_4 2024-09-14T03:30:27.564 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:30:27.564 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:30:27.564 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 977 Links: 1 2024-09-14T03:30:27.565 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:30:27.565 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:30:27.565 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-09-14 03:30:04.365732586 +0000 2024-09-14T03:30:27.565 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-09-14 03:26:08.791131953 +0000 2024-09-14T03:30:27.565 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-09-14 03:26:08.791131953 +0000 2024-09-14T03:30:27.565 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-09-14 03:26:08.791131953 +0000 2024-09-14T03:30:27.565 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:30:27.628 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-09-14T03:30:27.628 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-09-14T03:30:27.628 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 8.6854e-05 s, 5.9 MB/s 2024-09-14T03:30:27.629 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:30:27.686 INFO:tasks.cephadm:Deploying osd.0 on smithi110 with /dev/vg_nvme/lv_4... 2024-09-14T03:30:27.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-14T03:30:27.925 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:30:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: Deploying daemon mgr.x on smithi195 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[27365]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:30:27.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:30:27.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:30:27.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:27.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: Deploying daemon mgr.x on smithi195 2024-09-14T03:30:27.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:27 smithi110 ceph-mon[35374]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:27.952 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: Deploying daemon mgr.x on smithi195 2024-09-14T03:30:27.953 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:27 smithi195 ceph-mon[30695]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:29.113 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:28 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:28.981+0000 7f4f266cbec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:30:29.113 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:29.111+0000 7f4f266cbec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:30:29.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:29.258+0000 7f4f266cbec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:30:29.904 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:29.529+0000 7f4f266cbec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:30:29.904 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:29.593+0000 7f4f266cbec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:30:29.904 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:29.760+0000 7f4f266cbec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:30:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[27365]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[35374]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:29 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:30.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-mon[30695]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:30.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:29 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:30:30.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi110:vg_nvme/lv_4 2024-09-14T03:30:30.958 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:30.616+0000 7f4f266cbec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:30:30.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:30.680+0000 7f4f266cbec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:30:31.007 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:30:31.119 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.119 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.119 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:31.119 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[27365]: Reconfiguring daemon mgr.y on smithi110 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:31.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.121 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:30:31.121 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:30:31.121 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:30:31.121 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:31.121 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:31 smithi110 ceph-mon[35374]: Reconfiguring daemon mgr.y on smithi110 2024-09-14T03:30:31.386 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.386 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.387 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:31.387 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:31.387 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:31.387 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:30:31.387 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:30:31.387 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:30:31.387 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:31.387 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-mon[30695]: Reconfiguring daemon mgr.y on smithi110 2024-09-14T03:30:31.674 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:31.385+0000 7f4f266cbec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:30:31.674 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:31.495+0000 7f4f266cbec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:30:31.674 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:31.615+0000 7f4f266cbec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:30:31.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:31.673+0000 7f4f266cbec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:30:31.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:31.796+0000 7f4f266cbec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:30:31.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:31.861+0000 7f4f266cbec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:30:31.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:31 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:31.919+0000 7f4f266cbec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:30:32.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:32 smithi195 ceph-mon[30695]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:32.323 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:32 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:32.053+0000 7f4f266cbec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:30:32.323 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:32 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:32.116+0000 7f4f266cbec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:30:32.323 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:32 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:32.183+0000 7f4f266cbec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:30:32.323 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:32 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:32.265+0000 7f4f266cbec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:30:32.323 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:32 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:32.321+0000 7f4f266cbec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:30:32.418 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:32 smithi110 ceph-mon[27365]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:32.419 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:32 smithi110 ceph-mon[35374]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:32.708 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:32 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:32.379+0000 7f4f266cbec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:30:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: Standby manager daemon x started 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:30:33.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:30:33.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:30:33.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:33.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:33.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:33.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:30:33.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:30:33.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:33.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: Standby manager daemon x started 2024-09-14T03:30:33.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:30:33.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:30:33.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:30:33.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:33 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:30:33.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:30:33.113+0000 7f4f266cbec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:30:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:30:33.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:33.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: Standby manager daemon x started 2024-09-14T03:30:33.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:30:33.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:30:33.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:30:33.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:33 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/618436996' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:30:34.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:34 smithi110 ceph-mon[27365]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:34.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:34 smithi110 ceph-mon[27365]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:34.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:34 smithi110 ceph-mon[27365]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-14T03:30:34.422 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:30:34.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:34 smithi110 ceph-mon[35374]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:34.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:34 smithi110 ceph-mon[35374]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:34.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:34 smithi110 ceph-mon[35374]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-14T03:30:34.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:34 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:30:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:34 smithi195 ceph-mon[30695]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:34 smithi195 ceph-mon[30695]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:34 smithi195 ceph-mon[30695]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-14T03:30:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:34 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:30:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[35374]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/303207266' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]: dispatch 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]: dispatch 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]': finished 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[35374]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1246318965' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[27365]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/303207266' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]: dispatch 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]: dispatch 2024-09-14T03:30:36.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]': finished 2024-09-14T03:30:36.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[27365]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:30:36.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:36.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:36 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1246318965' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:30:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:36 smithi195 ceph-mon[30695]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:36 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/303207266' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]: dispatch 2024-09-14T03:30:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:36 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]: dispatch 2024-09-14T03:30:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:36 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "72194dcf-0a09-4942-8ff9-82510a5a8011"}]': finished 2024-09-14T03:30:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:36 smithi195 ceph-mon[30695]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:30:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:36 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:36 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1246318965' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:30:38.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:38 smithi110 ceph-mon[27365]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:38.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:38 smithi110 ceph-mon[35374]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:38.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:38 smithi195 ceph-mon[30695]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:40.431 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:40 smithi110 ceph-mon[27365]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:40.431 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:40 smithi110 ceph-mon[35374]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:40.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:40 smithi195 ceph-mon[30695]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:42 smithi110 ceph-mon[27365]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:42 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:30:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:42 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:42 smithi110 ceph-mon[27365]: Deploying daemon osd.0 on smithi110 2024-09-14T03:30:42.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:42 smithi110 ceph-mon[35374]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:42.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:42 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:30:42.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:42 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:42.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:42 smithi110 ceph-mon[35374]: Deploying daemon osd.0 on smithi110 2024-09-14T03:30:42.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:42 smithi195 ceph-mon[30695]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:42.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:42 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:30:42.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:42 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:42.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:42 smithi195 ceph-mon[30695]: Deploying daemon osd.0 on smithi110 2024-09-14T03:30:44.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:44 smithi110 ceph-mon[27365]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:44.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:44 smithi110 ceph-mon[35374]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:44.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:44 smithi195 ceph-mon[30695]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:46 smithi110 ceph-mon[35374]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:46 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:46 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:46 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:46.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:46 smithi110 ceph-mon[27365]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:46.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:46 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:46.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:46 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:46.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:46 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:46.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:46 smithi195 ceph-mon[30695]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:46.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:46 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:46.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:46 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:46.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:46 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:48.319 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:48 smithi110 ceph-mon[27365]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:48.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:48 smithi110 ceph-mon[35374]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:48.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:48 smithi195 ceph-mon[30695]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:49.412 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 0 on host 'smithi110' 2024-09-14T03:30:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[27365]: from='osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:30:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[27365]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:30:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[35374]: from='osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:30:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:49 smithi110 ceph-mon[35374]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:30:49.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:49 smithi195 ceph-mon[30695]: from='osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:30:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:49 smithi195 ceph-mon[30695]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:30:49.998 DEBUG:teuthology.orchestra.run.smithi110:osd.0> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.0.service 2024-09-14T03:30:50.002 INFO:tasks.cephadm:Deploying osd.1 on smithi110 with /dev/vg_nvme/lv_3... 2024-09-14T03:30:50.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-14T03:30:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:30:50.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:30:50.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:50.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: from='osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:30:50.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[35374]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:30:50.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:50.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:50.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:50.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:50.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:30:50.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:30:50.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:50.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: from='osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:30:50.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-mon[27365]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: from='osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:30:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:50 smithi195 ceph-mon[30695]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:30:51.140 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[39236]: 2024-09-14T03:30:50.829+0000 7ff362672640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T03:30:51.140 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:30:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[39236]: 2024-09-14T03:30:50.831+0000 7ff35f4ad640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:30:51.199 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:30:52.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[27365]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:52.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[27365]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:30:52.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[27365]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:30:52.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:52.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:52.007 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[35374]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:52.008 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[35374]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:30:52.008 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[35374]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:30:52.008 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:52.008 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:51 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:51 smithi195 ceph-mon[30695]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:30:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:51 smithi195 ceph-mon[30695]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:30:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:51 smithi195 ceph-mon[30695]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:30:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:51 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:51 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:52.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[27365]: purged_snaps scrub starts 2024-09-14T03:30:52.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[27365]: purged_snaps scrub ok 2024-09-14T03:30:52.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[27365]: osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602] boot 2024-09-14T03:30:52.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[27365]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:30:52.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:52.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[35374]: purged_snaps scrub starts 2024-09-14T03:30:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[35374]: purged_snaps scrub ok 2024-09-14T03:30:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[35374]: osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602] boot 2024-09-14T03:30:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[35374]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:30:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:52 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:52 smithi195 ceph-mon[30695]: purged_snaps scrub starts 2024-09-14T03:30:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:52 smithi195 ceph-mon[30695]: purged_snaps scrub ok 2024-09-14T03:30:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:52 smithi195 ceph-mon[30695]: osd.0 [v2:172.21.15.110:6802/1137454602,v1:172.21.15.110:6803/1137454602] boot 2024-09-14T03:30:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:52 smithi195 ceph-mon[30695]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:30:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:52 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:30:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:30:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: Detected new or changed devices on smithi110 2024-09-14T03:30:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi110 to 16064M 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: Detected new or changed devices on smithi110 2024-09-14T03:30:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:30:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi110 to 16064M 2024-09-14T03:30:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: Detected new or changed devices on smithi110 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi110 to 16064M 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:30:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:30:54.559 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:30:54.578 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi110:vg_nvme/lv_3 2024-09-14T03:30:54.815 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:30:56.066 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:55 smithi110 ceph-mon[27365]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:56.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:55 smithi110 ceph-mon[35374]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:56.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:55 smithi195 ceph-mon[30695]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:57.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:30:57.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:30:57.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:57.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:56 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:30:57.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:56 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:30:57.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:56 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:56 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:30:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:56 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:30:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:56 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:30:58.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:57 smithi110 ceph-mon[35374]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:58.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:57 smithi110 ceph-mon[35374]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:58.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:57 smithi110 ceph-mon[27365]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:58.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:57 smithi110 ceph-mon[27365]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:58.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:57 smithi195 ceph-mon[30695]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:30:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:57 smithi195 ceph-mon[30695]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:30:59.185 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:58 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1394337736' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91257255-3d84-4da6-b336-e878b7a021b0"}]: dispatch 2024-09-14T03:30:59.185 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:58 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1394337736' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91257255-3d84-4da6-b336-e878b7a021b0"}]': finished 2024-09-14T03:30:59.185 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:58 smithi110 ceph-mon[27365]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:30:59.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:58 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:30:59.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:58 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1394337736' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91257255-3d84-4da6-b336-e878b7a021b0"}]: dispatch 2024-09-14T03:30:59.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:58 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1394337736' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91257255-3d84-4da6-b336-e878b7a021b0"}]': finished 2024-09-14T03:30:59.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:58 smithi110 ceph-mon[35374]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:30:59.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:58 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:30:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:58 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1394337736' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91257255-3d84-4da6-b336-e878b7a021b0"}]: dispatch 2024-09-14T03:30:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:58 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1394337736' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91257255-3d84-4da6-b336-e878b7a021b0"}]': finished 2024-09-14T03:30:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:58 smithi195 ceph-mon[30695]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:30:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:58 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:59 smithi110 ceph-mon[35374]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:30:59 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2776991777' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:00.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:59 smithi110 ceph-mon[27365]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:00.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:30:59 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2776991777' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:00.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:59 smithi195 ceph-mon[30695]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:30:59 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2776991777' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:01.997 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:01 smithi110 ceph-mon[27365]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:01.997 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:01 smithi110 ceph-mon[35374]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:01 smithi195 ceph-mon[30695]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:04.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:03 smithi110 ceph-mon[27365]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:03 smithi110 ceph-mon[35374]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:04.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:03 smithi195 ceph-mon[30695]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:05.137 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:31:05.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:05.138 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:31:05.138 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:31:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:06.052 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:05 smithi110 ceph-mon[27365]: Deploying daemon osd.1 on smithi110 2024-09-14T03:31:06.053 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:05 smithi110 ceph-mon[27365]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:06.053 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:05 smithi110 ceph-mon[35374]: Deploying daemon osd.1 on smithi110 2024-09-14T03:31:06.053 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:05 smithi110 ceph-mon[35374]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:05 smithi195 ceph-mon[30695]: Deploying daemon osd.1 on smithi110 2024-09-14T03:31:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:05 smithi195 ceph-mon[30695]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:07.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:07 smithi110 ceph-mon[27365]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:07.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:07 smithi110 ceph-mon[35374]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:08.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:07 smithi195 ceph-mon[30695]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:10.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:09 smithi110 ceph-mon[27365]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:10.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:09 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:10.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:09 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:10.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:09 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:10.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:09 smithi110 ceph-mon[35374]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:10.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:09 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:10.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:09 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:10.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:09 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:10.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:09 smithi195 ceph-mon[30695]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:09 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:09 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:09 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[35374]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[27365]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:11 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:11 smithi195 ceph-mon[30695]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:11 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:11 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:11 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:11 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:11 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:12.437 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 1 on host 'smithi110' 2024-09-14T03:31:13.038 DEBUG:teuthology.orchestra.run.smithi110:osd.1> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.1.service 2024-09-14T03:31:13.041 INFO:tasks.cephadm:Deploying osd.2 on smithi110 with /dev/vg_nvme/lv_2... 2024-09-14T03:31:13.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-14T03:31:13.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:13.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:13.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:13.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:12 smithi110 ceph-mon[27365]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:31:13.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:13.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:13.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:13.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:12 smithi110 ceph-mon[35374]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:31:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:12 smithi195 ceph-mon[30695]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:31:13.716 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:31:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[27365]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[27365]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:31:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[27365]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:31:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[27365]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:31:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[35374]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[35374]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:31:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[35374]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:31:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:13 smithi110 ceph-mon[35374]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:31:14.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:13 smithi195 ceph-mon[30695]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:13 smithi195 ceph-mon[30695]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:31:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:13 smithi195 ceph-mon[30695]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:31:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:13 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:13 smithi195 ceph-mon[30695]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:31:14.696 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[45099]: 2024-09-14T03:31:14.441+0000 7f6346799640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T03:31:14.696 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[45099]: 2024-09-14T03:31:14.445+0000 7f63435d4640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:31:15.551 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[27365]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:31:15.551 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[27365]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:31:15.551 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:15.551 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:15.551 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[27365]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:15.551 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[35374]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:31:15.551 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[35374]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:31:15.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:15.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:15.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:15 smithi110 ceph-mon[35374]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:15 smithi195 ceph-mon[30695]: from='osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:31:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:15 smithi195 ceph-mon[30695]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:31:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:15 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:15 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:15 smithi195 ceph-mon[30695]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:31:16.141 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:31:16.160 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi110:vg_nvme/lv_2 2024-09-14T03:31:16.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[27365]: purged_snaps scrub starts 2024-09-14T03:31:16.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[27365]: purged_snaps scrub ok 2024-09-14T03:31:16.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[27365]: osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487] boot 2024-09-14T03:31:16.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[27365]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:31:16.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:16.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[35374]: purged_snaps scrub starts 2024-09-14T03:31:16.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[35374]: purged_snaps scrub ok 2024-09-14T03:31:16.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[35374]: osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487] boot 2024-09-14T03:31:16.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[35374]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:31:16.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:16 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:16.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:16 smithi195 ceph-mon[30695]: purged_snaps scrub starts 2024-09-14T03:31:16.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:16 smithi195 ceph-mon[30695]: purged_snaps scrub ok 2024-09-14T03:31:16.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:16 smithi195 ceph-mon[30695]: osd.1 [v2:172.21.15.110:6810/3479666487,v1:172.21.15.110:6811/3479666487] boot 2024-09-14T03:31:16.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:16 smithi195 ceph-mon[30695]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:31:16.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:16 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:31:16.757 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:31:17.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:17 smithi110 ceph-mon[27365]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:31:17.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:17 smithi110 ceph-mon[27365]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:17.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:17 smithi110 ceph-mon[35374]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:31:17.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:17 smithi110 ceph-mon[35374]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:17.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:17 smithi195 ceph-mon[30695]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:31:17.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:17 smithi195 ceph-mon[30695]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:19.691 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: Detected new or changed devices on smithi110 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi110 to 8032M 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:31:19.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: Detected new or changed devices on smithi110 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi110 to 8032M 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:19.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:19.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:19.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:31:19.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:31:19.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:31:19.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:19 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: Detected new or changed devices on smithi110 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi110 to 8032M 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:19.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:19.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:19.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:31:19.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:31:19.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:31:19.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:19 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:22.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[27365]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3284781611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]: dispatch 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]: dispatch 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]': finished 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[27365]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/97510590' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[35374]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3284781611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]: dispatch 2024-09-14T03:31:22.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]: dispatch 2024-09-14T03:31:22.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]': finished 2024-09-14T03:31:22.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[35374]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:31:22.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:22.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:21 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/97510590' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:21 smithi195 ceph-mon[30695]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:21 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3284781611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]: dispatch 2024-09-14T03:31:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:21 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]: dispatch 2024-09-14T03:31:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:21 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d"}]': finished 2024-09-14T03:31:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:21 smithi195 ceph-mon[30695]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:31:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:21 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/97510590' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:24.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:23 smithi110 ceph-mon[27365]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:24.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:23 smithi110 ceph-mon[35374]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:23 smithi195 ceph-mon[30695]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:26.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:25 smithi110 ceph-mon[27365]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:26.156 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:25 smithi110 ceph-mon[35374]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:26.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:25 smithi195 ceph-mon[30695]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:27 smithi110 ceph-mon[27365]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:31:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:27 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:27 smithi110 ceph-mon[27365]: Deploying daemon osd.2 on smithi110 2024-09-14T03:31:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:27 smithi110 ceph-mon[35374]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:31:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:27 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:27 smithi110 ceph-mon[35374]: Deploying daemon osd.2 on smithi110 2024-09-14T03:31:28.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:27 smithi195 ceph-mon[30695]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:31:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:27 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:27 smithi195 ceph-mon[30695]: Deploying daemon osd.2 on smithi110 2024-09-14T03:31:29.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:29 smithi110 ceph-mon[27365]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:29.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:29 smithi110 ceph-mon[35374]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:30.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:29 smithi195 ceph-mon[30695]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:32.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:31 smithi110 ceph-mon[35374]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:32.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:31 smithi110 ceph-mon[27365]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:32.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:31 smithi195 ceph-mon[30695]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:32.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:32.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:32.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:32 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:32.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:32 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:32.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:32 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:32.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:32 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:32 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:32 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:32 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:34 smithi110 ceph-mon[27365]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:34 smithi110 ceph-mon[35374]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:34 smithi195 ceph-mon[30695]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:35.338 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 2 on host 'smithi110' 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:35.969 DEBUG:teuthology.orchestra.run.smithi110:osd.2> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.2.service 2024-09-14T03:31:35.971 INFO:tasks.cephadm:Deploying osd.3 on smithi110 with /dev/vg_nvme/lv_1... 2024-09-14T03:31:35.971 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-14T03:31:36.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-mon[35374]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:31:36.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-mon[35374]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:31:36.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:36.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-mon[35374]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:31:36.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-mon[27365]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:31:36.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-mon[27365]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:31:36.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:36.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-mon[27365]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:31:36.934 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[50653]: 2024-09-14T03:31:36.712+0000 7f525511a640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T03:31:36.934 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:36 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[50653]: 2024-09-14T03:31:36.715+0000 7f524ff51640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:31:36.987 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:31:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:36 smithi195 ceph-mon[30695]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:31:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:36 smithi195 ceph-mon[30695]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:31:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:36 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:36 smithi195 ceph-mon[30695]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[35374]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[35374]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[35374]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[27365]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[27365]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:37.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:37.908 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:37 smithi110 ceph-mon[27365]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:37 smithi195 ceph-mon[30695]: from='osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:31:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:37 smithi195 ceph-mon[30695]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:31:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:37 smithi195 ceph-mon[30695]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:31:38.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[27365]: purged_snaps scrub starts 2024-09-14T03:31:38.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[27365]: purged_snaps scrub ok 2024-09-14T03:31:38.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[27365]: osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173] boot 2024-09-14T03:31:38.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[27365]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:31:38.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:38.805 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[35374]: purged_snaps scrub starts 2024-09-14T03:31:38.805 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[35374]: purged_snaps scrub ok 2024-09-14T03:31:38.805 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[35374]: osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173] boot 2024-09-14T03:31:38.806 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[35374]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:31:38.806 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:38 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:38 smithi195 ceph-mon[30695]: purged_snaps scrub starts 2024-09-14T03:31:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:38 smithi195 ceph-mon[30695]: purged_snaps scrub ok 2024-09-14T03:31:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:38 smithi195 ceph-mon[30695]: osd.2 [v2:172.21.15.110:6818/3061361173,v1:172.21.15.110:6819/3061361173] boot 2024-09-14T03:31:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:38 smithi195 ceph-mon[30695]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:31:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:38 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: pgmap v72: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: Detected new or changed devices on smithi110 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi110 to 5354M 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: pgmap v72: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: Detected new or changed devices on smithi110 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.147 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi110 to 5354M 2024-09-14T03:31:40.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:40.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:40.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:39 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:31:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: pgmap v72: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-14T03:31:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: Detected new or changed devices on smithi110 2024-09-14T03:31:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:31:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi110 to 5354M 2024-09-14T03:31:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:31:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:39 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:40.401 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:31:40.421 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi110:vg_nvme/lv_1 2024-09-14T03:31:40.661 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:31:41.007 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:40 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-14T03:31:41.007 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:40 smithi110 ceph-mon[35374]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:31:41.007 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:40 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:31:41.007 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:40 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-14T03:31:41.007 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:40 smithi110 ceph-mon[27365]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:31:41.007 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:40 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:31:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:40 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-14T03:31:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:40 smithi195 ceph-mon[30695]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:31:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:40 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:31:41.268 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54795]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:31:41.268 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54795]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:41.268 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54795]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:41.268 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54795]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:41.532 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54811]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:31:41.532 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54811]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:41.532 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54811]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:41.532 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54811]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:41.792 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:41 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:31:41.793 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:41 smithi110 ceph-mon[35374]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:31:41.793 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:41 smithi110 ceph-mon[35374]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:41.793 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:41 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:41.793 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:41 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:31:41.793 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:41 smithi110 ceph-mon[27365]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:31:41.793 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:41 smithi110 ceph-mon[27365]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:41.793 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:41 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:41.794 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54815]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:31:41.794 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54815]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:41.794 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54815]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:41.794 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54815]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:42.051 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54822]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:31:42.051 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54822]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:42.051 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54822]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:42.051 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:31:41 smithi110 sudo[54822]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:42.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:41 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:31:42.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:41 smithi195 ceph-mon[30695]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:31:42.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:41 smithi195 ceph-mon[30695]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:42.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:41 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:42.313 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54848]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:31:42.314 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54848]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:42.314 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54848]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:42.314 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54848]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:42.577 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54872]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:31:42.577 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54872]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:42.577 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54872]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:42.577 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54872]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:42.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54876]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:31:42.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54876]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:42.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54876]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:42.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:42 smithi110 ceph-mon[35374]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:31:42.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:42 smithi110 ceph-mon[35374]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:31:42.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54876]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:42.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54879]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:31:42.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54879]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:42.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54879]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 sudo[54879]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 ceph-mon[27365]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:31:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:42 smithi110 ceph-mon[27365]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:31:43.083 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:42 smithi195 ceph-mon[30695]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:31:43.083 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:42 smithi195 ceph-mon[30695]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:31:43.083 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 sudo[33174]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:31:43.083 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 sudo[33174]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:43.083 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 sudo[33174]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:43.083 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 sudo[33174]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:43.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 sudo[33177]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:31:43.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 sudo[33177]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:43.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 sudo[33177]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:43.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 sudo[33177]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:43.860 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 sudo[55021]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:31:43.860 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 sudo[55021]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:43.860 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 sudo[55021]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:43.860 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 sudo[55021]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:31:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:44.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:44.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:44.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 sudo[55025]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:31:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 sudo[55025]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:31:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 sudo[55025]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 sudo[55025]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:44.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:31:44.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:44.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:44.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:44.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:31:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:31:44.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:31:44.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:31:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:44 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:44 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:44 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:44 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:44 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:31:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:44 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:31:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[27365]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2275067135' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]: dispatch 2024-09-14T03:31:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]: dispatch 2024-09-14T03:31:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]': finished 2024-09-14T03:31:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[27365]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:31:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:31:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1862406778' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:46.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[35374]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:46.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2275067135' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]: dispatch 2024-09-14T03:31:46.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]: dispatch 2024-09-14T03:31:46.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]': finished 2024-09-14T03:31:46.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[35374]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:31:46.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:31:46.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:45 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1862406778' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:45 smithi195 ceph-mon[30695]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:45 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2275067135' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]: dispatch 2024-09-14T03:31:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:45 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]: dispatch 2024-09-14T03:31:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:45 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "38519016-56eb-4523-b648-fe57a8bc35b9"}]': finished 2024-09-14T03:31:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:45 smithi195 ceph-mon[30695]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:31:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:45 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:31:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:45 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1862406778' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:31:48.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:47 smithi110 ceph-mon[27365]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:48.166 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:47 smithi110 ceph-mon[35374]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:47 smithi195 ceph-mon[30695]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:50.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:49 smithi110 ceph-mon[27365]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:49 smithi110 ceph-mon[35374]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:49 smithi195 ceph-mon[30695]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:52.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:51 smithi110 ceph-mon[35374]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:52.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:51 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:31:52.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:51 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:52.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:51 smithi110 ceph-mon[35374]: Deploying daemon osd.3 on smithi110 2024-09-14T03:31:52.161 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:51 smithi110 ceph-mon[27365]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:52.161 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:51 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:31:52.161 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:51 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:52.162 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:51 smithi110 ceph-mon[27365]: Deploying daemon osd.3 on smithi110 2024-09-14T03:31:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:51 smithi195 ceph-mon[30695]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:51 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:31:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:51 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:31:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:51 smithi195 ceph-mon[30695]: Deploying daemon osd.3 on smithi110 2024-09-14T03:31:54.154 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:53 smithi110 ceph-mon[27365]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:54.154 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:53 smithi110 ceph-mon[35374]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:53 smithi195 ceph-mon[30695]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:55 smithi110 ceph-mon[35374]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:55 smithi110 ceph-mon[27365]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:56.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:55 smithi195 ceph-mon[30695]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:57.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:56 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:57.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:56 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:57.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:56 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:57.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:56 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:57.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:56 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:31:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:56 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:56 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:31:58.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:31:58 smithi110 ceph-mon[35374]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:58.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:31:58 smithi110 ceph-mon[27365]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:58.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:31:58 smithi195 ceph-mon[30695]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:31:59.355 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 3 on host 'smithi110' 2024-09-14T03:31:59.985 DEBUG:teuthology.orchestra.run.smithi110:osd.3> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.3.service 2024-09-14T03:31:59.987 INFO:tasks.cephadm:Deploying osd.4 on smithi195 with /dev/vg_nvme/lv_4... 2024-09-14T03:31:59.987 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-14T03:32:00.239 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:32:00.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:00 smithi195 ceph-mon[30695]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:32:00.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:00 smithi195 ceph-mon[30695]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:32:00.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:00 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:00.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:00 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:00.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:00 smithi110 ceph-mon[27365]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:32:00.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:00 smithi110 ceph-mon[27365]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:32:00.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:00 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:00.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:00 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:00.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:00 smithi110 ceph-mon[35374]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:32:00.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:00 smithi110 ceph-mon[35374]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:32:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:00 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:00 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:01.080 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[35374]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:32:01.080 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[35374]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:32:01.080 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[35374]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[27365]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[27365]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[27365]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:01.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:01.082 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[56794]: 2024-09-14T03:32:01.026+0000 7f63132bb640 -1 osd.3 0 waiting for initial osdmap 2024-09-14T03:32:01.082 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:32:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[56794]: 2024-09-14T03:32:01.030+0000 7f630a8c9640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:32:01.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:01 smithi195 ceph-mon[30695]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:32:01.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:01 smithi195 ceph-mon[30695]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:32:01.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:01 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:01.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:01 smithi195 ceph-mon[30695]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-14T03:32:01.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:01 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:01.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:01 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:02.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:02 smithi195 ceph-mon[30695]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:32:02.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:02 smithi195 ceph-mon[30695]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:32:02.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:02 smithi195 ceph-mon[30695]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:32:02.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:02 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:02.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:02 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:02.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[27365]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:32:02.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[27365]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:32:02.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[27365]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:32:02.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:02.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:02.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[35374]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:32:02.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[35374]: from='osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-14T03:32:02.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[35374]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:32:02.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:02.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:02 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:02.604 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:32:02.623 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi195:vg_nvme/lv_4 2024-09-14T03:32:02.881 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:32:03.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[35374]: purged_snaps scrub starts 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[35374]: purged_snaps scrub ok 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[35374]: osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960] boot 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[35374]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[27365]: purged_snaps scrub starts 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[27365]: purged_snaps scrub ok 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[27365]: osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960] boot 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[27365]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:32:03.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:03 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:03.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:03 smithi195 ceph-mon[30695]: purged_snaps scrub starts 2024-09-14T03:32:03.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:03 smithi195 ceph-mon[30695]: purged_snaps scrub ok 2024-09-14T03:32:03.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:03 smithi195 ceph-mon[30695]: osd.3 [v2:172.21.15.110:6826/3810786960,v1:172.21.15.110:6827/3810786960] boot 2024-09-14T03:32:03.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:03 smithi195 ceph-mon[30695]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:32:03.254 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:03 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:32:04.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:04.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:32:04.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: Detected new or changed devices on smithi110 2024-09-14T03:32:04.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi110 to 4016M 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:04.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:04 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:32:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: Detected new or changed devices on smithi110 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi110 to 4016M 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:04.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:04.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:04.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:32:04.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: Detected new or changed devices on smithi110 2024-09-14T03:32:04.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi110 to 4016M 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:04.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:04 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:05.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:05 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:05.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:05 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:05.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:05 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:05 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:05 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:05 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:05 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:05.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:05 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:05.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:05 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:06.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:06 smithi195 ceph-mon[30695]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:06.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:06 smithi195 ceph-mon[30695]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:06 smithi110 ceph-mon[27365]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:06 smithi110 ceph-mon[27365]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:06.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:06 smithi110 ceph-mon[35374]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:06.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:06 smithi110 ceph-mon[35374]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:07.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/2732917245' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]: dispatch 2024-09-14T03:32:07.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]: dispatch 2024-09-14T03:32:07.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]': finished 2024-09-14T03:32:07.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[27365]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:32:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:07.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/2732917245' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]: dispatch 2024-09-14T03:32:07.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]: dispatch 2024-09-14T03:32:07.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]': finished 2024-09-14T03:32:07.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[35374]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:32:07.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:07 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:07.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:07 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/2732917245' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]: dispatch 2024-09-14T03:32:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:07 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]: dispatch 2024-09-14T03:32:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:07 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76641e65-6c1a-4adc-9c67-e6c89dc248e4"}]': finished 2024-09-14T03:32:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:07 smithi195 ceph-mon[30695]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:32:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:07 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:08.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:08 smithi110 ceph-mon[27365]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:08.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:08 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/4168431718' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:08.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:08 smithi110 ceph-mon[35374]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:08.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:08 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/4168431718' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:08.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:08 smithi195 ceph-mon[30695]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:08.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:08 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/4168431718' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:10.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:10 smithi110 ceph-mon[27365]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:10.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:10 smithi110 ceph-mon[35374]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:10.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:10 smithi195 ceph-mon[30695]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:12 smithi195 ceph-mon[30695]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:12.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:12 smithi110 ceph-mon[27365]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:12 smithi110 ceph-mon[35374]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:13.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:13 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:32:13.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:13 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:13.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:13 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:32:13.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:13 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:32:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:13 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:14.279 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:14 smithi195 ceph-mon[30695]: Deploying daemon osd.4 on smithi195 2024-09-14T03:32:14.279 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:14 smithi195 ceph-mon[30695]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:14 smithi110 ceph-mon[27365]: Deploying daemon osd.4 on smithi195 2024-09-14T03:32:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:14 smithi110 ceph-mon[27365]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:14 smithi110 ceph-mon[35374]: Deploying daemon osd.4 on smithi195 2024-09-14T03:32:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:14 smithi110 ceph-mon[35374]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:16.162 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:16 smithi195 ceph-mon[30695]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:16.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:16 smithi110 ceph-mon[27365]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:16 smithi110 ceph-mon[35374]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:18.263 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:18 smithi195 ceph-mon[30695]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:18.263 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:18.264 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:18.264 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:18 smithi110 ceph-mon[27365]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:18.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:18 smithi110 ceph-mon[35374]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:18.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:18 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:18.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:18 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:18.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:18 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:20.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:20 smithi195 ceph-mon[30695]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:20.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:20 smithi110 ceph-mon[27365]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:20.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:20 smithi110 ceph-mon[35374]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:20.968 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 4 on host 'smithi195' 2024-09-14T03:32:21.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:21 smithi195 ceph-mon[30695]: from='osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:32:21.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:21 smithi195 ceph-mon[30695]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:32:21.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:21 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.564 DEBUG:teuthology.orchestra.run.smithi195:osd.4> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.4.service 2024-09-14T03:32:21.566 INFO:tasks.cephadm:Deploying osd.5 on smithi195 with /dev/vg_nvme/lv_3... 2024-09-14T03:32:21.566 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-14T03:32:21.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[27365]: from='osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:32:21.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[27365]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:32:21.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[35374]: from='osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:32:21.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[35374]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:32:21.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:21.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:21 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:22.638 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:32:22.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[27365]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:22.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[27365]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:32:22.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[27365]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:32:22.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:22.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[27365]: from='osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:22.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[27365]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:22.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[35374]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:22.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[35374]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:32:22.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[35374]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:32:22.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:22.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[35374]: from='osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:22.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:22 smithi110 ceph-mon[35374]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:22.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:22 smithi195 ceph-mon[30695]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:22 smithi195 ceph-mon[30695]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:32:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:22 smithi195 ceph-mon[30695]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:32:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:22 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:22 smithi195 ceph-mon[30695]: from='osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:22 smithi195 ceph-mon[30695]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:22.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:32:22 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[35489]: 2024-09-14T03:32:22.328+0000 7f83f2f4f640 -1 osd.4 0 waiting for initial osdmap 2024-09-14T03:32:22.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:32:22 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[35489]: 2024-09-14T03:32:22.332+0000 7f83edd86640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:32:23.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:23 smithi110 ceph-mon[27365]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:32:23.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:23 smithi110 ceph-mon[27365]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:32:23.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:23 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:23.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:23 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:23 smithi110 ceph-mon[35374]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:32:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:23 smithi110 ceph-mon[35374]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:32:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:23 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:23.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:23 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:23.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:23 smithi195 ceph-mon[30695]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:32:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:23 smithi195 ceph-mon[30695]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:32:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:23 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:23.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:23 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:24.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:24 smithi195 ceph-mon[30695]: purged_snaps scrub starts 2024-09-14T03:32:24.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:24 smithi195 ceph-mon[30695]: purged_snaps scrub ok 2024-09-14T03:32:24.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:24 smithi195 ceph-mon[30695]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:24.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:24 smithi195 ceph-mon[30695]: osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350] boot 2024-09-14T03:32:24.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:24 smithi195 ceph-mon[30695]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:32:24.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:24 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:24.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[27365]: purged_snaps scrub starts 2024-09-14T03:32:24.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[27365]: purged_snaps scrub ok 2024-09-14T03:32:24.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[27365]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[27365]: osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350] boot 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[27365]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[35374]: purged_snaps scrub starts 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[35374]: purged_snaps scrub ok 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[35374]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[35374]: osd.4 [v2:172.21.15.195:6800/3499292350,v1:172.21.15.195:6801/3499292350] boot 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[35374]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:32:24.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:24 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:32:25.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:32:25.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:25.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:25.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:25.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:25.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:25.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:25.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:25 smithi195 ceph-mon[30695]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:32:25.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:32:25.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:25.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[27365]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:25.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:25 smithi110 ceph-mon[35374]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:32:26.007 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:32:26.027 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi195:vg_nvme/lv_3 2024-09-14T03:32:26.264 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:32:26.663 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:26 smithi195 ceph-mon[30695]: Detected new or changed devices on smithi195 2024-09-14T03:32:26.663 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:26 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi195 to 17088M 2024-09-14T03:32:26.664 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:26 smithi195 ceph-mon[30695]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:26.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:26 smithi110 ceph-mon[27365]: Detected new or changed devices on smithi195 2024-09-14T03:32:26.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:26 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi195 to 17088M 2024-09-14T03:32:26.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:26 smithi110 ceph-mon[27365]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:26.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:26 smithi110 ceph-mon[35374]: Detected new or changed devices on smithi195 2024-09-14T03:32:26.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:26 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi195 to 17088M 2024-09-14T03:32:26.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:26 smithi110 ceph-mon[35374]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:28 smithi195 ceph-mon[30695]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-09-14T03:32:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:28 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:28 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:28 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:28.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:28 smithi110 ceph-mon[27365]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-09-14T03:32:28.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:28 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:28.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:28 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:28.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:28 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:28.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:28 smithi110 ceph-mon[35374]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-09-14T03:32:28.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:28 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:28.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:28 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:28.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:28 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:29 smithi195 ceph-mon[30695]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:29.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:29 smithi110 ceph-mon[27365]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:29.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:29 smithi110 ceph-mon[35374]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:30.598 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[35374]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:32:30.598 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/1350510288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]: dispatch 2024-09-14T03:32:30.598 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]: dispatch 2024-09-14T03:32:30.599 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]': finished 2024-09-14T03:32:30.599 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[35374]: osdmap e34: 6 total, 5 up, 6 in 2024-09-14T03:32:30.599 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:30.599 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[27365]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:32:30.599 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/1350510288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]: dispatch 2024-09-14T03:32:30.599 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]: dispatch 2024-09-14T03:32:30.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]': finished 2024-09-14T03:32:30.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[27365]: osdmap e34: 6 total, 5 up, 6 in 2024-09-14T03:32:30.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:30 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:30.650 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:30 smithi195 ceph-mon[30695]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:32:30.650 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:30 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/1350510288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]: dispatch 2024-09-14T03:32:30.650 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:30 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]: dispatch 2024-09-14T03:32:30.650 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:30 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7"}]': finished 2024-09-14T03:32:30.650 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:30 smithi195 ceph-mon[30695]: osdmap e34: 6 total, 5 up, 6 in 2024-09-14T03:32:30.650 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:30 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:31.617 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:31 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/867822639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:31.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:31 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/867822639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:31.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:31 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/867822639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:32.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:32 smithi110 ceph-mon[27365]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:32:32.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:32 smithi110 ceph-mon[35374]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:32:32.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:32 smithi195 ceph-mon[30695]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:32:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:34 smithi195 ceph-mon[30695]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:32:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:34 smithi110 ceph-mon[27365]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:32:34.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:34 smithi110 ceph-mon[35374]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:32:36.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:36 smithi195 ceph-mon[30695]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:32:36.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:36 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:32:36.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:36 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:36 smithi110 ceph-mon[27365]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:32:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:36 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:32:36.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:36 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:36.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:36 smithi110 ceph-mon[35374]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:32:36.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:36 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:32:36.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:36 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:37.604 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:37 smithi195 ceph-mon[30695]: Deploying daemon osd.5 on smithi195 2024-09-14T03:32:37.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:37 smithi110 ceph-mon[27365]: Deploying daemon osd.5 on smithi195 2024-09-14T03:32:37.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:37 smithi110 ceph-mon[35374]: Deploying daemon osd.5 on smithi195 2024-09-14T03:32:38.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:38 smithi195 ceph-mon[30695]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:38.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:38 smithi110 ceph-mon[27365]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:38.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:38 smithi110 ceph-mon[35374]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:40.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:40 smithi195 ceph-mon[30695]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:40.632 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:40 smithi110 ceph-mon[35374]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:40.632 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:40 smithi110 ceph-mon[27365]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:41.619 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:41 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:41.619 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:41 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:41.619 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:41 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:41.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:41 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:41.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:41 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:41.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:41 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:41.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:41 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:41.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:41 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:41.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:41 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:42.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:42 smithi195 ceph-mon[30695]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:42.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:42 smithi110 ceph-mon[27365]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:42.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:42 smithi110 ceph-mon[35374]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:44.367 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 5 on host 'smithi195' 2024-09-14T03:32:44.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:44.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:44.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:44.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:32:44.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:32:44.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:44.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:44.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:44.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:44.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:32:44.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:32:44.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:44.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:44 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:44.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:44 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:45.063 DEBUG:teuthology.orchestra.run.smithi195:osd.5> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.5.service 2024-09-14T03:32:45.065 INFO:tasks.cephadm:Deploying osd.6 on smithi195 with /dev/vg_nvme/lv_2... 2024-09-14T03:32:45.066 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-14T03:32:45.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[27365]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:32:45.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[27365]: osdmap e35: 6 total, 5 up, 6 in 2024-09-14T03:32:45.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:45.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[27365]: from='osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:45.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[27365]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:45.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[27365]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:45.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[35374]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:32:45.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[35374]: osdmap e35: 6 total, 5 up, 6 in 2024-09-14T03:32:45.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:45.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[35374]: from='osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:45.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[35374]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:45.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:45 smithi110 ceph-mon[35374]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:45.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:45 smithi195 ceph-mon[30695]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:32:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:45 smithi195 ceph-mon[30695]: osdmap e35: 6 total, 5 up, 6 in 2024-09-14T03:32:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:45 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:45 smithi195 ceph-mon[30695]: from='osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:45 smithi195 ceph-mon[30695]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:32:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:45 smithi195 ceph-mon[30695]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:32:45.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:32:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[41452]: 2024-09-14T03:32:45.684+0000 7f2d5091a640 -1 osd.5 0 waiting for initial osdmap 2024-09-14T03:32:45.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:32:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[41452]: 2024-09-14T03:32:45.690+0000 7f2d4d755640 -1 osd.5 36 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:32:46.140 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:32:46.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:46 smithi195 ceph-mon[30695]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:32:46.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:46 smithi195 ceph-mon[30695]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:32:46.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:46 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:46.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:46 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:46.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:46 smithi110 ceph-mon[27365]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:32:46.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:46 smithi110 ceph-mon[27365]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:32:46.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:46 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:46.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:46 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:46.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:46 smithi110 ceph-mon[35374]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:32:46.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:46 smithi110 ceph-mon[35374]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:32:46.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:46 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:46.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:46 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:47.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[27365]: purged_snaps scrub starts 2024-09-14T03:32:47.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[27365]: purged_snaps scrub ok 2024-09-14T03:32:47.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[27365]: osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662] boot 2024-09-14T03:32:47.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[27365]: osdmap e37: 6 total, 6 up, 6 in 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[27365]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[27365]: osdmap e38: 6 total, 6 up, 6 in 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[35374]: purged_snaps scrub starts 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[35374]: purged_snaps scrub ok 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[35374]: osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662] boot 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[35374]: osdmap e37: 6 total, 6 up, 6 in 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[35374]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:47.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:47 smithi110 ceph-mon[35374]: osdmap e38: 6 total, 6 up, 6 in 2024-09-14T03:32:47.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:47 smithi195 ceph-mon[30695]: purged_snaps scrub starts 2024-09-14T03:32:47.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:47 smithi195 ceph-mon[30695]: purged_snaps scrub ok 2024-09-14T03:32:47.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:47 smithi195 ceph-mon[30695]: osd.5 [v2:172.21.15.195:6808/4151750662,v1:172.21.15.195:6809/4151750662] boot 2024-09-14T03:32:47.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:47 smithi195 ceph-mon[30695]: osdmap e37: 6 total, 6 up, 6 in 2024-09-14T03:32:47.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:47 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:32:47.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:47 smithi195 ceph-mon[30695]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:47.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:47 smithi195 ceph-mon[30695]: osdmap e38: 6 total, 6 up, 6 in 2024-09-14T03:32:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: Detected new or changed devices on smithi195 2024-09-14T03:32:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:49.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi195 to 8544M 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[27365]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: Detected new or changed devices on smithi195 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:49.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:49.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi195 to 8544M 2024-09-14T03:32:49.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:49.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:49.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:49 smithi110 ceph-mon[35374]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:32:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: Detected new or changed devices on smithi195 2024-09-14T03:32:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:32:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi195 to 8544M 2024-09-14T03:32:49.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:49.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:32:49.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:32:49.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:49 smithi195 ceph-mon[30695]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:32:49.808 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:32:49.827 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi195:vg_nvme/lv_2 2024-09-14T03:32:50.067 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:32:50.457 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:50 smithi195 ceph-mon[30695]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:50.457 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:50 smithi195 ceph-mon[30695]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:32:50.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:50 smithi110 ceph-mon[27365]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:50.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:50 smithi110 ceph-mon[27365]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:32:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:50 smithi110 ceph-mon[35374]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:50 smithi110 ceph-mon[35374]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:32:52.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:52 smithi195 ceph-mon[30695]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:52.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:52 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:52.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:52 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:52.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:52 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:52.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:52 smithi110 ceph-mon[27365]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:52.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:52 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:52.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:52 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:52.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:52 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:52.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:52 smithi110 ceph-mon[35374]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:52.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:52 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:32:52.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:52 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:32:52.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:52 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:32:53.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:53 smithi110 ceph-mon[27365]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:53.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:53 smithi110 ceph-mon[35374]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:53 smithi195 ceph-mon[30695]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:32:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:54 smithi195 ceph-mon[30695]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:54 smithi195 ceph-mon[30695]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:32:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:54 smithi195 ceph-mon[30695]: Cluster is now healthy 2024-09-14T03:32:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:54 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/3652161642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]: dispatch 2024-09-14T03:32:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:54 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]: dispatch 2024-09-14T03:32:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:54 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]': finished 2024-09-14T03:32:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:54 smithi195 ceph-mon[30695]: osdmap e40: 7 total, 6 up, 7 in 2024-09-14T03:32:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:54 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:32:54.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[27365]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:54.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[27365]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:32:54.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[27365]: Cluster is now healthy 2024-09-14T03:32:54.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/3652161642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]: dispatch 2024-09-14T03:32:54.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]: dispatch 2024-09-14T03:32:54.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]': finished 2024-09-14T03:32:54.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[27365]: osdmap e40: 7 total, 6 up, 7 in 2024-09-14T03:32:54.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:32:54.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[35374]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:32:54.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[35374]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:32:54.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[35374]: Cluster is now healthy 2024-09-14T03:32:54.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/3652161642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]: dispatch 2024-09-14T03:32:54.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]: dispatch 2024-09-14T03:32:54.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc7c808d-655e-4183-b18f-f6b57a6a5797"}]': finished 2024-09-14T03:32:54.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[35374]: osdmap e40: 7 total, 6 up, 7 in 2024-09-14T03:32:54.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:54 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:32:55.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:55 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/4080228076' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:55.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:55 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/4080228076' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:55.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:55 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/4080228076' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:32:56.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:56 smithi110 ceph-mon[27365]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-14T03:32:56.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:56 smithi110 ceph-mon[35374]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-14T03:32:56.706 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:56 smithi195 ceph-mon[30695]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-14T03:32:58.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:32:58 smithi195 ceph-mon[30695]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-14T03:32:58.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:32:58 smithi110 ceph-mon[27365]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-14T03:32:58.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:32:58 smithi110 ceph-mon[35374]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-14T03:33:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:00 smithi195 ceph-mon[30695]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:33:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:00 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:33:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:00 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:00.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:00 smithi110 ceph-mon[35374]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:33:00.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:00 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:33:00.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:00 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:00.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:00 smithi110 ceph-mon[27365]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:33:00.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:00 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:33:00.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:00 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:01.694 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:01 smithi195 ceph-mon[30695]: Deploying daemon osd.6 on smithi195 2024-09-14T03:33:01.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:01 smithi110 ceph-mon[35374]: Deploying daemon osd.6 on smithi195 2024-09-14T03:33:01.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:01 smithi110 ceph-mon[27365]: Deploying daemon osd.6 on smithi195 2024-09-14T03:33:02.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:02 smithi110 ceph-mon[27365]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:33:02.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:02 smithi110 ceph-mon[35374]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:33:02.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:02 smithi195 ceph-mon[30695]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:33:04.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:04 smithi195 ceph-mon[30695]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:04.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:04 smithi110 ceph-mon[27365]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:04.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:04 smithi110 ceph-mon[35374]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:05 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:05 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:05 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:05 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:05 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:05 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:05 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:05 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:05 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:06.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:06 smithi110 ceph-mon[27365]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:06 smithi110 ceph-mon[35374]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:06.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:06 smithi195 ceph-mon[30695]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:08.194 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 6 on host 'smithi195' 2024-09-14T03:33:08.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:08.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:08.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:08.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:33:08.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:33:08.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:08.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:08.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:08.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:08.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:33:08.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:33:08.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:08.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:08 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:08.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:08 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:08.827 DEBUG:teuthology.orchestra.run.smithi195:osd.6> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.6.service 2024-09-14T03:33:08.830 INFO:tasks.cephadm:Deploying osd.7 on smithi195 with /dev/vg_nvme/lv_1... 2024-09-14T03:33:08.830 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-14T03:33:09.606 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:33:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:09 smithi195 ceph-mon[30695]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:33:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:09 smithi195 ceph-mon[30695]: osdmap e41: 7 total, 6 up, 7 in 2024-09-14T03:33:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:09 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:09 smithi195 ceph-mon[30695]: from='osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:09 smithi195 ceph-mon[30695]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:09 smithi195 ceph-mon[30695]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:09.710 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:33:09 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[47115]: 2024-09-14T03:33:09.505+0000 7faf8e4a1640 -1 osd.6 0 waiting for initial osdmap 2024-09-14T03:33:09.710 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:33:09 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[47115]: 2024-09-14T03:33:09.510+0000 7faf892d8640 -1 osd.6 42 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:33:09.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[27365]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:33:09.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[27365]: osdmap e41: 7 total, 6 up, 7 in 2024-09-14T03:33:09.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:09.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[27365]: from='osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:09.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[27365]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:09.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[27365]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:09.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[35374]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:33:09.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[35374]: osdmap e41: 7 total, 6 up, 7 in 2024-09-14T03:33:09.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:09.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[35374]: from='osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:09.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[35374]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:09.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:09 smithi110 ceph-mon[35374]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:33:10.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:10 smithi195 ceph-mon[30695]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:33:10.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:10 smithi195 ceph-mon[30695]: osdmap e42: 7 total, 6 up, 7 in 2024-09-14T03:33:10.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:10 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:10.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:10 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:10.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:10 smithi195 ceph-mon[30695]: osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795] boot 2024-09-14T03:33:10.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:10 smithi195 ceph-mon[30695]: osdmap e43: 7 total, 7 up, 7 in 2024-09-14T03:33:10.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:10 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[27365]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:33:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[27365]: osdmap e42: 7 total, 6 up, 7 in 2024-09-14T03:33:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[27365]: osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795] boot 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[27365]: osdmap e43: 7 total, 7 up, 7 in 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[35374]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[35374]: osdmap e42: 7 total, 6 up, 7 in 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[35374]: osd.6 [v2:172.21.15.195:6816/258227795,v1:172.21.15.195:6817/258227795] boot 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[35374]: osdmap e43: 7 total, 7 up, 7 in 2024-09-14T03:33:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:10 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:33:12.349 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:11 smithi195 ceph-mon[30695]: purged_snaps scrub starts 2024-09-14T03:33:12.349 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:11 smithi195 ceph-mon[30695]: purged_snaps scrub ok 2024-09-14T03:33:12.349 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:11 smithi195 ceph-mon[30695]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:12.349 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:11 smithi195 ceph-mon[30695]: osdmap e44: 7 total, 7 up, 7 in 2024-09-14T03:33:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:11 smithi110 ceph-mon[35374]: purged_snaps scrub starts 2024-09-14T03:33:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:11 smithi110 ceph-mon[35374]: purged_snaps scrub ok 2024-09-14T03:33:12.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:11 smithi110 ceph-mon[35374]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:12.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:11 smithi110 ceph-mon[35374]: osdmap e44: 7 total, 7 up, 7 in 2024-09-14T03:33:12.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:11 smithi110 ceph-mon[27365]: purged_snaps scrub starts 2024-09-14T03:33:12.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:11 smithi110 ceph-mon[27365]: purged_snaps scrub ok 2024-09-14T03:33:12.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:11 smithi110 ceph-mon[27365]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:12.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:11 smithi110 ceph-mon[27365]: osdmap e44: 7 total, 7 up, 7 in 2024-09-14T03:33:13.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: osdmap e45: 7 total, 7 up, 7 in 2024-09-14T03:33:13.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:13.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:13.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:12 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: osdmap e45: 7 total, 7 up, 7 in 2024-09-14T03:33:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: osdmap e45: 7 total, 7 up, 7 in 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:13.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:13.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:13.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:13.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:12 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:14.129 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-14T03:33:14.148 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi195:vg_nvme/lv_1 2024-09-14T03:33:14.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:13 smithi195 ceph-mon[30695]: Detected new or changed devices on smithi195 2024-09-14T03:33:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:13 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi195 to 5696M 2024-09-14T03:33:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:13 smithi195 ceph-mon[30695]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:14.390 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:33:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:13 smithi110 ceph-mon[27365]: Detected new or changed devices on smithi195 2024-09-14T03:33:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:13 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi195 to 5696M 2024-09-14T03:33:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:13 smithi110 ceph-mon[27365]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:13 smithi110 ceph-mon[35374]: Detected new or changed devices on smithi195 2024-09-14T03:33:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:13 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi195 to 5696M 2024-09-14T03:33:14.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:13 smithi110 ceph-mon[35374]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:16.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:15 smithi195 ceph-mon[30695]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:16.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:15 smithi110 ceph-mon[27365]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:15 smithi110 ceph-mon[35374]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:16 smithi195 ceph-mon[30695]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:33:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:16 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:33:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:16 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:33:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:16 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:17.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:16 smithi110 ceph-mon[27365]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:33:17.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:16 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:33:17.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:16 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:33:17.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:16 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:16 smithi110 ceph-mon[35374]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:33:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:16 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:33:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:16 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:33:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:16 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:18.168 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:17 smithi195 ceph-mon[30695]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:17 smithi110 ceph-mon[27365]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:17 smithi110 ceph-mon[35374]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/1582611568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]: dispatch 2024-09-14T03:33:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]: dispatch 2024-09-14T03:33:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[27365]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]': finished 2024-09-14T03:33:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[27365]: osdmap e46: 8 total, 7 up, 8 in 2024-09-14T03:33:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/1582611568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]: dispatch 2024-09-14T03:33:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]: dispatch 2024-09-14T03:33:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[35374]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]': finished 2024-09-14T03:33:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[35374]: osdmap e46: 8 total, 7 up, 8 in 2024-09-14T03:33:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:18 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:19.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:18 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/1582611568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]: dispatch 2024-09-14T03:33:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:18 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]: dispatch 2024-09-14T03:33:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:18 smithi195 ceph-mon[30695]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7b35908-a28c-4497-a688-ac61826e57ac"}]': finished 2024-09-14T03:33:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:18 smithi195 ceph-mon[30695]: osdmap e46: 8 total, 7 up, 8 in 2024-09-14T03:33:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:18 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:20.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:19 smithi110 ceph-mon[27365]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:20.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:19 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/2604849037' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:33:20.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:19 smithi110 ceph-mon[35374]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:20.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:19 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/2604849037' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:33:20.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:19 smithi195 ceph-mon[30695]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:20.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:19 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/2604849037' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:33:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:21 smithi195 ceph-mon[30695]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:22.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:21 smithi110 ceph-mon[27365]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:22.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:21 smithi110 ceph-mon[35374]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:24.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:23 smithi195 ceph-mon[30695]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:24.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:23 smithi110 ceph-mon[27365]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:24.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:23 smithi110 ceph-mon[35374]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:25.134 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:24 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:33:25.135 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:24 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:25.135 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:24 smithi195 ceph-mon[30695]: Deploying daemon osd.7 on smithi195 2024-09-14T03:33:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:24 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:33:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:24 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:24 smithi110 ceph-mon[35374]: Deploying daemon osd.7 on smithi195 2024-09-14T03:33:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:24 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:33:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:24 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:24 smithi110 ceph-mon[27365]: Deploying daemon osd.7 on smithi195 2024-09-14T03:33:26.273 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:25 smithi195 ceph-mon[30695]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:26.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:25 smithi110 ceph-mon[27365]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:26.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:25 smithi110 ceph-mon[35374]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:27 smithi110 ceph-mon[35374]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:28.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:27 smithi110 ceph-mon[27365]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:28.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:27 smithi195 ceph-mon[30695]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:30.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:29 smithi195 ceph-mon[30695]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:30.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:29 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:30.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:29 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:30.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:29 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:30.248 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:29 smithi110 ceph-mon[35374]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:30.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:29 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:30.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:29 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:30.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:29 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:30.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:29 smithi110 ceph-mon[27365]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:30.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:29 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:30.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:29 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:30.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:29 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:32.299 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:31 smithi195 ceph-mon[30695]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:31 smithi110 ceph-mon[27365]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:32.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:31 smithi110 ceph-mon[35374]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:33.034 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 7 on host 'smithi195' 2024-09-14T03:33:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[27365]: from='osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:33:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[27365]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:33:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[35374]: from='osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:33:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[35374]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:33:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.702 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.702 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.702 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:33 smithi195 ceph-mon[30695]: from='osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:33:33.702 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:33 smithi195 ceph-mon[30695]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:33:33.702 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.703 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:33.726 DEBUG:teuthology.orchestra.run.smithi195:osd.7> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.7.service 2024-09-14T03:33:33.729 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-14T03:33:33.729 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-14T03:33:33.983 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:33:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[27365]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[27365]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:33:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[27365]: osdmap e47: 8 total, 7 up, 8 in 2024-09-14T03:33:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[27365]: from='osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[27365]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[35374]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[35374]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:33:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[35374]: osdmap e47: 8 total, 7 up, 8 in 2024-09-14T03:33:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[35374]: from='osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:34 smithi110 ceph-mon[35374]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:34.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:34 smithi195 ceph-mon[30695]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:34.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:34 smithi195 ceph-mon[30695]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:33:34.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:34 smithi195 ceph-mon[30695]: osdmap e47: 8 total, 7 up, 8 in 2024-09-14T03:33:34.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:34 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:34.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:34 smithi195 ceph-mon[30695]: from='osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:34.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:34 smithi195 ceph-mon[30695]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-14T03:33:34.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:33:34 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[53109]: 2024-09-14T03:33:34.371+0000 7f505f2d5640 -1 osd.7 0 waiting for initial osdmap 2024-09-14T03:33:34.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:33:34 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[53109]: 2024-09-14T03:33:34.378+0000 7f505b90f640 -1 osd.7 48 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:33:35.651 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[35374]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:33:35.651 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[35374]: osdmap e48: 8 total, 7 up, 8 in 2024-09-14T03:33:35.651 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.651 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.651 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[35374]: osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944] boot 2024-09-14T03:33:35.651 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[35374]: osdmap e49: 8 total, 8 up, 8 in 2024-09-14T03:33:35.652 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[27365]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:33:35.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[27365]: osdmap e48: 8 total, 7 up, 8 in 2024-09-14T03:33:35.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[27365]: osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944] boot 2024-09-14T03:33:35.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[27365]: osdmap e49: 8 total, 8 up, 8 in 2024-09-14T03:33:35.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:35 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:35 smithi195 ceph-mon[30695]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-14T03:33:35.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:35 smithi195 ceph-mon[30695]: osdmap e48: 8 total, 7 up, 8 in 2024-09-14T03:33:35.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:35 smithi195 ceph-mon[30695]: osd.7 [v2:172.21.15.195:6824/284932944,v1:172.21.15.195:6825/284932944] boot 2024-09-14T03:33:35.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:35 smithi195 ceph-mon[30695]: osdmap e49: 8 total, 8 up, 8 in 2024-09-14T03:33:35.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:35 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:33:35.808 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:33:36.481 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":49,"num_osds":8,"num_up_osds":8,"osd_up_since":1726284814,"num_in_osds":8,"osd_in_since":1726284798,"num_remapped_pgs":1} 2024-09-14T03:33:36.482 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:33:36.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:36 smithi195 ceph-mon[30695]: purged_snaps scrub starts 2024-09-14T03:33:36.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:36 smithi195 ceph-mon[30695]: purged_snaps scrub ok 2024-09-14T03:33:36.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:36 smithi195 ceph-mon[30695]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:36.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:36 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1091727464' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:33:36.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:36 smithi195 ceph-mon[30695]: osdmap e50: 8 total, 8 up, 8 in 2024-09-14T03:33:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[27365]: purged_snaps scrub starts 2024-09-14T03:33:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[27365]: purged_snaps scrub ok 2024-09-14T03:33:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[27365]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1091727464' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:33:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[27365]: osdmap e50: 8 total, 8 up, 8 in 2024-09-14T03:33:36.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[35374]: purged_snaps scrub starts 2024-09-14T03:33:36.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[35374]: purged_snaps scrub ok 2024-09-14T03:33:36.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[35374]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:33:36.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1091727464' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:33:36.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:36 smithi110 ceph-mon[35374]: osdmap e50: 8 total, 8 up, 8 in 2024-09-14T03:33:36.747 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:33:37.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: Detected new or changed devices on smithi195 2024-09-14T03:33:37.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi195 to 4272M 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[27365]: osdmap e51: 8 total, 8 up, 8 in 2024-09-14T03:33:37.931 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: Detected new or changed devices on smithi195 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi195 to 4272M 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:37.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:37.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:37.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:37.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:37 smithi110 ceph-mon[35374]: osdmap e51: 8 total, 8 up, 8 in 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: Detected new or changed devices on smithi195 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi195 to 4272M 2024-09-14T03:33:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:37.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:37 smithi195 ceph-mon[30695]: osdmap e51: 8 total, 8 up, 8 in 2024-09-14T03:33:38.549 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:33:38.549 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":51,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:33:36.962693+0000","last_up_change":"2024-09-14T03:33:34.960002+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6803","nonce":1137454602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6805","nonce":1137454602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6809","nonce":1137454602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6807","nonce":1137454602}]},"public_addr":"172.21.15.110:6803/1137454602","cluster_addr":"172.21.15.110:6805/1137454602","heartbeat_back_addr":"172.21.15.110:6809/1137454602","heartbeat_front_addr":"172.21.15.110:6807/1137454602","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6811","nonce":3479666487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6813","nonce":3479666487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6817","nonce":3479666487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6815","nonce":3479666487}]},"public_addr":"172.21.15.110:6811/3479666487","cluster_addr":"172.21.15.110:6813/3479666487","heartbeat_back_addr":"172.21.15.110:6817/3479666487","heartbeat_front_addr":"172.21.15.110:6815/3479666487","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6819","nonce":3061361173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6821","nonce":3061361173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6825","nonce":3061361173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6823","nonce":3061361173}]},"public_addr":"172.21.15.110:6819/3061361173","cluster_addr":"172.21.15.110:6821/3061361173","heartbeat_back_addr":"172.21.15.110:6825/3061361173","heartbeat_front_addr":"172.21.15.110:6823/3061361173","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6827","nonce":3810786960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6829","nonce":3810786960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6833","nonce":3810786960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6831","nonce":3810786960}]},"public_addr":"172.21.15.110:6827/3810786960","cluster_addr":"172.21.15.110:6829/3810786960","heartbeat_back_addr":"172.21.15.110:6833/3810786960","heartbeat_front_addr":"172.21.15.110:6831/3810786960","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6801","nonce":3499292350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6803","nonce":3499292350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6807","nonce":3499292350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6805","nonce":3499292350}]},"public_addr":"172.21.15.195:6801/3499292350","cluster_addr":"172.21.15.195:6803/3499292350","heartbeat_back_addr":"172.21.15.195:6807/3499292350","heartbeat_front_addr":"172.21.15.195:6805/3499292350","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6809","nonce":4151750662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6811","nonce":4151750662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6815","nonce":4151750662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6813","nonce":4151750662}]},"public_addr":"172.21.15.195:6809/4151750662","cluster_addr":"172.21.15.195:6811/4151750662","heartbeat_back_addr":"172.21.15.195:6815/4151750662","heartbeat_front_addr":"172.21.15.195:6813/4151750662","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6817","nonce":258227795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6819","nonce":258227795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6823","nonce":258227795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6821","nonce":258227795}]},"public_addr":"172.21.15.195:6817/258227795","cluster_addr":"172.21.15.195:6819/258227795","heartbeat_back_addr":"172.21.15.195:6823/258227795","heartbeat_front_addr":"172.21.15.195:6821/258227795","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6825","nonce":284932944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6827","nonce":284932944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6831","nonce":284932944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6829","nonce":284932944}]},"public_addr":"172.21.15.195:6825/284932944","cluster_addr":"172.21.15.195:6827/284932944","heartbeat_back_addr":"172.21.15.195:6831/284932944","heartbeat_front_addr":"172.21.15.195:6829/284932944","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:33:38.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:38 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3321878470' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:33:38.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:38 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3321878470' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:33:39.162 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-14T03:31:39.074672+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-14T03:33:39.162 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-14T03:33:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:38 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3321878470' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:33:39.403 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:33:39.815 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:39 smithi110 ceph-mon[27365]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:39.816 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:39 smithi110 ceph-mon[35374]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:39 smithi195 ceph-mon[30695]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:41.179 INFO:teuthology.orchestra.run.smithi110.stdout:pg_num: 1 2024-09-14T03:33:41.818 INFO:tasks.cephadm:Adding mds.a on smithi110 2024-09-14T03:33:41.818 INFO:tasks.cephadm:Adding mds.b on smithi195 2024-09-14T03:33:41.818 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch apply mds all '2;smithi110=a;smithi195=b' 2024-09-14T03:33:42.087 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:33:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:41 smithi110 ceph-mon[27365]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:41 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/147221281' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:33:42.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:41 smithi110 ceph-mon[35374]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:42.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:41 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/147221281' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:33:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:41 smithi195 ceph-mon[30695]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:41 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/147221281' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:33:43.773 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled mds.all update... 2024-09-14T03:33:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:33:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:33:44.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:43 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:44.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:33:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:33:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:43 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:44.530 DEBUG:teuthology.orchestra.run.smithi110:mds.a> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.a.service 2024-09-14T03:33:44.533 DEBUG:teuthology.orchestra.run.smithi195:mds.b> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.b.service 2024-09-14T03:33:44.535 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-09-14T03:33:44.536 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:33:45.154 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:33:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:44 smithi110 ceph-mon[27365]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi110=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:33:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:44 smithi110 ceph-mon[27365]: Saving service mds.all spec with placement smithi110=a;smithi195=b;count:2 2024-09-14T03:33:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:44 smithi110 ceph-mon[27365]: Deploying daemon mds.a on smithi110 2024-09-14T03:33:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:44 smithi110 ceph-mon[35374]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi110=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:33:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:44 smithi110 ceph-mon[35374]: Saving service mds.all spec with placement smithi110=a;smithi195=b;count:2 2024-09-14T03:33:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:44 smithi110 ceph-mon[35374]: Deploying daemon mds.a on smithi110 2024-09-14T03:33:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:44 smithi195 ceph-mon[30695]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi110=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:33:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:44 smithi195 ceph-mon[30695]: Saving service mds.all spec with placement smithi110=a;smithi195=b;count:2 2024-09-14T03:33:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:44 smithi195 ceph-mon[30695]: Deploying daemon mds.a on smithi110 2024-09-14T03:33:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:45 smithi110 ceph-mon[27365]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:45 smithi110 ceph-mon[35374]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:46.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:45 smithi195 ceph-mon[30695]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:47.152 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:33:47.153 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24311,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.110:6835/2015946497","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6834","nonce":2015946497},{"type":"v1","addr":"172.21.15.110:6835","nonce":2015946497}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2024-09-14T03:33:47.153 INFO:teuthology.orchestra.run.smithi110.stderr:dumped fsmap epoch 2 2024-09-14T03:33:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: mds.? [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] up:boot 2024-09-14T03:33:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: fsmap 1 up:standby 2024-09-14T03:33:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:33:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2148331719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:33:48.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:33:48.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:33:48.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:48.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:48 smithi195 ceph-mon[30695]: Deploying daemon mds.b on smithi195 2024-09-14T03:33:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: mds.? [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] up:boot 2024-09-14T03:33:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: fsmap 1 up:standby 2024-09-14T03:33:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:33:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2148331719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:33:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[27365]: Deploying daemon mds.b on smithi195 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: mds.? [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] up:boot 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: fsmap 1 up:standby 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:33:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2148331719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:33:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:33:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:33:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:48 smithi110 ceph-mon[35374]: Deploying daemon mds.b on smithi195 2024-09-14T03:33:49.209 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:33:49 smithi195 podman[57006]: 2024-09-14 03:33:49.170661708 +0000 UTC m=+0.132216070 container create d306b18ee20b211b6443f08ba70f6cd57bdc8a9c73367fb3fe379c093680ea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux ) 2024-09-14T03:33:49.709 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:33:49 smithi195 podman[57006]: 2024-09-14 03:33:49.398183575 +0000 UTC m=+0.359737934 container init d306b18ee20b211b6443f08ba70f6cd57bdc8a9c73367fb3fe379c093680ea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-14T03:33:49.709 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:33:49 smithi195 podman[57006]: 2024-09-14 03:33:49.400927134 +0000 UTC m=+0.362481496 container start d306b18ee20b211b6443f08ba70f6cd57bdc8a9c73367fb3fe379c093680ea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-eb16b97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:33:49.709 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:33:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b[57021]: starting mds.b at 2024-09-14T03:33:49.709 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:33:49 smithi195 bash[57006]: d306b18ee20b211b6443f08ba70f6cd57bdc8a9c73367fb3fe379c093680ea16 2024-09-14T03:33:49.709 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:33:49 smithi195 systemd[1]: Started Ceph mds.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:33:49.759 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:33:50.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[27365]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:50.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.361 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:50.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[35374]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:50.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:50 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:50.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:50 smithi195 ceph-mon[30695]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:50.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:50.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:50 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:50.628 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:33:51.387 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[35374]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:boot 2024-09-14T03:33:51.387 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[35374]: fsmap 2 up:standby 2024-09-14T03:33:51.387 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:33:51.387 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:51.387 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:51.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[27365]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:boot 2024-09-14T03:33:51.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[27365]: fsmap 2 up:standby 2024-09-14T03:33:51.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:33:51.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:51.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:51 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:51.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:51 smithi195 ceph-mon[30695]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:boot 2024-09-14T03:33:51.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:51 smithi195 ceph-mon[30695]: fsmap 2 up:standby 2024-09-14T03:33:51.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:51 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:33:51.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:51 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:51.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:51 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:52.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:52 smithi110 ceph-mon[35374]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:52.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:52 smithi110 ceph-mon[27365]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:52.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:52 smithi195 ceph-mon[30695]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:52.470 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:33:52.471 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24310,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.195:6833/1180864653","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6832","nonce":1180864653},{"type":"v1","addr":"172.21.15.195:6833","nonce":1180864653}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24311,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.110:6835/2015946497","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6834","nonce":2015946497},{"type":"v1","addr":"172.21.15.110:6835","nonce":2015946497}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2024-09-14T03:33:52.471 INFO:teuthology.orchestra.run.smithi110.stderr:dumped fsmap epoch 3 2024-09-14T03:33:53.107 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph --format=json-pretty osd lspools 2024-09-14T03:33:53.348 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:33:53.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:53.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:53.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:53.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:53.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/824991649' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:53.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:53.646 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.646 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/824991649' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:33:53.646 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:53 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/824991649' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:33:53.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:53 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:33:54.669 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:54 smithi110 ceph-mon[35374]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:54.670 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:54 smithi110 ceph-mon[27365]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:54.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:54 smithi195 ceph-mon[30695]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:55.004 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:33:55.004 INFO:teuthology.orchestra.run.smithi110.stdout:[ 2024-09-14T03:33:55.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2024-09-14T03:33:55.004 INFO:teuthology.orchestra.run.smithi110.stdout: "poolnum": 1, 2024-09-14T03:33:55.005 INFO:teuthology.orchestra.run.smithi110.stdout: "poolname": ".mgr" 2024-09-14T03:33:55.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:33:55.005 INFO:teuthology.orchestra.run.smithi110.stdout:] 2024-09-14T03:33:55.656 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:55 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2660475928' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-14T03:33:55.656 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:55 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2660475928' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-14T03:33:55.657 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-09-14T03:33:55.657 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool create cephfs_metadata --pg_num_min 64 2024-09-14T03:33:55.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:55 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2660475928' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-14T03:33:55.901 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:33:56.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:56 smithi110 ceph-mon[35374]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:56.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:56 smithi110 ceph-mon[27365]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:56.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:56 smithi195 ceph-mon[30695]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:58.392 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'cephfs_metadata' created 2024-09-14T03:33:58.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:58 smithi110 ceph-mon[27365]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:58.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:58 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/780372079' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:33:58.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:58 smithi110 ceph-mon[35374]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:58.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:58 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/780372079' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:33:58.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:58 smithi195 ceph-mon[30695]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:33:58.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:58 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/780372079' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:33:59.107 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-09-14T03:33:59.351 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:33:59.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:59 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/780372079' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-14T03:33:59.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:59 smithi110 ceph-mon[27365]: osdmap e52: 8 total, 8 up, 8 in 2024-09-14T03:33:59.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:33:59 smithi110 ceph-mon[27365]: osdmap e53: 8 total, 8 up, 8 in 2024-09-14T03:33:59.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:59 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/780372079' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-14T03:33:59.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:59 smithi110 ceph-mon[35374]: osdmap e52: 8 total, 8 up, 8 in 2024-09-14T03:33:59.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:33:59 smithi110 ceph-mon[35374]: osdmap e53: 8 total, 8 up, 8 in 2024-09-14T03:33:59.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:59 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/780372079' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-14T03:33:59.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:59 smithi195 ceph-mon[30695]: osdmap e52: 8 total, 8 up, 8 in 2024-09-14T03:33:59.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:33:59 smithi195 ceph-mon[30695]: osdmap e53: 8 total, 8 up, 8 in 2024-09-14T03:34:00.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:00 smithi110 ceph-mon[27365]: pgmap v175: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:00.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:00 smithi110 ceph-mon[27365]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:00.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:00 smithi110 ceph-mon[27365]: osdmap e54: 8 total, 8 up, 8 in 2024-09-14T03:34:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:00 smithi110 ceph-mon[35374]: pgmap v175: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:00 smithi110 ceph-mon[35374]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:00 smithi110 ceph-mon[35374]: osdmap e54: 8 total, 8 up, 8 in 2024-09-14T03:34:00.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:00 smithi195 ceph-mon[30695]: pgmap v175: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:00.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:00 smithi195 ceph-mon[30695]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:00.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:00 smithi195 ceph-mon[30695]: osdmap e54: 8 total, 8 up, 8 in 2024-09-14T03:34:01.493 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'cephfs_data' created 2024-09-14T03:34:01.786 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:01 smithi110 ceph-mon[35374]: pgmap v178: 33 pgs: 18 active+clean, 15 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:01.786 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:01 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/4155598331' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:34:01.786 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:01 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:34:01.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:01 smithi110 ceph-mon[27365]: pgmap v178: 33 pgs: 18 active+clean, 15 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:01.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:01 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4155598331' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:34:01.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:01 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:34:01.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:01 smithi195 ceph-mon[30695]: pgmap v178: 33 pgs: 18 active+clean, 15 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:01.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:01 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/4155598331' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:34:01.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:01 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:34:02.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs new cephfs cephfs_metadata cephfs_data 2024-09-14T03:34:02.367 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:02.793 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:02 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-14T03:34:02.794 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:02 smithi110 ceph-mon[35374]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:34:02.794 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:02 smithi110 ceph-mon[35374]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:34:02.794 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:02 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-14T03:34:02.794 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:02 smithi110 ceph-mon[27365]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:34:02.794 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:02 smithi110 ceph-mon[27365]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:34:02.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:02 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-14T03:34:02.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:02 smithi195 ceph-mon[30695]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:34:02.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:02 smithi195 ceph-mon[30695]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:34:03.746 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:03 smithi110 ceph-mon[35374]: pgmap v181: 97 pgs: 33 creating+peering, 34 active+clean, 30 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:03.747 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:03 smithi110 ceph-mon[27365]: pgmap v181: 97 pgs: 33 creating+peering, 34 active+clean, 30 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:03.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:03 smithi195 ceph-mon[30695]: pgmap v181: 97 pgs: 33 creating+peering, 34 active+clean, 30 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:04.512 INFO:teuthology.orchestra.run.smithi110.stderr:new fs with metadata pool 2 and data pool 3 2024-09-14T03:34:04.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:04 smithi110 ceph-mon[35374]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:34:04.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:04 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1919975310' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:34:04.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:04 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:34:04.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:04 smithi110 ceph-mon[35374]: daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:34:04.814 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:04 smithi110 ceph-mon[27365]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:34:04.814 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:04 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1919975310' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:34:04.814 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:04 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:34:04.814 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:04 smithi110 ceph-mon[27365]: daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:34:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:04 smithi195 ceph-mon[30695]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:34:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:04 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1919975310' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:34:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:04 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:34:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:04 smithi195 ceph-mon[30695]: daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:34:05.142 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:34:05.388 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:05.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[35374]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:34:05.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-14T03:34:05.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[35374]: fsmap cephfs:1 {0=a=up:creating} 1 up:standby 2024-09-14T03:34:05.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[35374]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:34:05.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[35374]: pgmap v184: 97 pgs: 33 creating+peering, 40 active+clean, 24 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:05.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[35374]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:05.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[35374]: mds.? [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] up:active 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[35374]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[27365]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[27365]: fsmap cephfs:1 {0=a=up:creating} 1 up:standby 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[27365]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[27365]: pgmap v184: 97 pgs: 33 creating+peering, 40 active+clean, 24 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[27365]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[27365]: mds.? [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] up:active 2024-09-14T03:34:05.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:05 smithi110 ceph-mon[27365]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:05 smithi195 ceph-mon[30695]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:34:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:05 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-14T03:34:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:05 smithi195 ceph-mon[30695]: fsmap cephfs:1 {0=a=up:creating} 1 up:standby 2024-09-14T03:34:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:05 smithi195 ceph-mon[30695]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:34:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:05 smithi195 ceph-mon[30695]: pgmap v184: 97 pgs: 33 creating+peering, 40 active+clean, 24 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:05 smithi195 ceph-mon[30695]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:05 smithi195 ceph-mon[30695]: mds.? [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] up:active 2024-09-14T03:34:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:05 smithi195 ceph-mon[30695]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:06.861 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:06 smithi110 ceph-mon[35374]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-14T03:34:06.862 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:06 smithi110 ceph-mon[35374]: Cluster is now healthy 2024-09-14T03:34:06.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:06 smithi110 ceph-mon[27365]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-14T03:34:06.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:06 smithi110 ceph-mon[27365]: Cluster is now healthy 2024-09-14T03:34:06.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:06 smithi195 ceph-mon[30695]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-14T03:34:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:06 smithi195 ceph-mon[30695]: Cluster is now healthy 2024-09-14T03:34:07.196 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:34:07 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[61551]: 2024-09-14T03:34:07.088+0000 7fb8f042b640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-14T03:34:07.204 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:34:07.204 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":58,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:34:04.160250+0000","last_up_change":"2024-09-14T03:33:34.960002+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:33:57.860054+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:34:01.436541+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6803","nonce":1137454602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6805","nonce":1137454602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6809","nonce":1137454602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6807","nonce":1137454602}]},"public_addr":"172.21.15.110:6803/1137454602","cluster_addr":"172.21.15.110:6805/1137454602","heartbeat_back_addr":"172.21.15.110:6809/1137454602","heartbeat_front_addr":"172.21.15.110:6807/1137454602","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6811","nonce":3479666487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6813","nonce":3479666487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6817","nonce":3479666487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6815","nonce":3479666487}]},"public_addr":"172.21.15.110:6811/3479666487","cluster_addr":"172.21.15.110:6813/3479666487","heartbeat_back_addr":"172.21.15.110:6817/3479666487","heartbeat_front_addr":"172.21.15.110:6815/3479666487","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6819","nonce":3061361173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6821","nonce":3061361173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6825","nonce":3061361173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6823","nonce":3061361173}]},"public_addr":"172.21.15.110:6819/3061361173","cluster_addr":"172.21.15.110:6821/3061361173","heartbeat_back_addr":"172.21.15.110:6825/3061361173","heartbeat_front_addr":"172.21.15.110:6823/3061361173","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6827","nonce":3810786960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6829","nonce":3810786960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6833","nonce":3810786960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6831","nonce":3810786960}]},"public_addr":"172.21.15.110:6827/3810786960","cluster_addr":"172.21.15.110:6829/3810786960","heartbeat_back_addr":"172.21.15.110:6833/3810786960","heartbeat_front_addr":"172.21.15.110:6831/3810786960","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6801","nonce":3499292350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6803","nonce":3499292350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6807","nonce":3499292350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6805","nonce":3499292350}]},"public_addr":"172.21.15.195:6801/3499292350","cluster_addr":"172.21.15.195:6803/3499292350","heartbeat_back_addr":"172.21.15.195:6807/3499292350","heartbeat_front_addr":"172.21.15.195:6805/3499292350","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6809","nonce":4151750662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6811","nonce":4151750662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6815","nonce":4151750662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6813","nonce":4151750662}]},"public_addr":"172.21.15.195:6809/4151750662","cluster_addr":"172.21.15.195:6811/4151750662","heartbeat_back_addr":"172.21.15.195:6815/4151750662","heartbeat_front_addr":"172.21.15.195:6813/4151750662","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6817","nonce":258227795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6819","nonce":258227795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6823","nonce":258227795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6821","nonce":258227795}]},"public_addr":"172.21.15.195:6817/258227795","cluster_addr":"172.21.15.195:6819/258227795","heartbeat_back_addr":"172.21.15.195:6823/258227795","heartbeat_front_addr":"172.21.15.195:6821/258227795","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6825","nonce":284932944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6827","nonce":284932944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6831","nonce":284932944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6829","nonce":284932944}]},"public_addr":"172.21.15.195:6825/284932944","cluster_addr":"172.21.15.195:6827/284932944","heartbeat_back_addr":"172.21.15.195:6831/284932944","heartbeat_front_addr":"172.21.15.195:6829/284932944","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:34:07.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:07 smithi110 ceph-mon[27365]: pgmap v185: 97 pgs: 21 creating+peering, 76 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 958 B/s wr, 3 op/s 2024-09-14T03:34:07.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:07 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2064339712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:07.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:07 smithi110 ceph-mon[35374]: pgmap v185: 97 pgs: 21 creating+peering, 76 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 958 B/s wr, 3 op/s 2024-09-14T03:34:07.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:07 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2064339712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:07.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:07 smithi195 ceph-mon[30695]: pgmap v185: 97 pgs: 21 creating+peering, 76 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 958 B/s wr, 3 op/s 2024-09-14T03:34:07.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:07 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2064339712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:07.973 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:34:08.218 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:09.874 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:34:09.875 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":58,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:34:04.160250+0000","last_up_change":"2024-09-14T03:33:34.960002+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:33:57.860054+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:34:01.436541+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6803","nonce":1137454602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6805","nonce":1137454602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6809","nonce":1137454602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6807","nonce":1137454602}]},"public_addr":"172.21.15.110:6803/1137454602","cluster_addr":"172.21.15.110:6805/1137454602","heartbeat_back_addr":"172.21.15.110:6809/1137454602","heartbeat_front_addr":"172.21.15.110:6807/1137454602","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6811","nonce":3479666487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6813","nonce":3479666487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6817","nonce":3479666487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6815","nonce":3479666487}]},"public_addr":"172.21.15.110:6811/3479666487","cluster_addr":"172.21.15.110:6813/3479666487","heartbeat_back_addr":"172.21.15.110:6817/3479666487","heartbeat_front_addr":"172.21.15.110:6815/3479666487","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6819","nonce":3061361173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6821","nonce":3061361173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6825","nonce":3061361173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6823","nonce":3061361173}]},"public_addr":"172.21.15.110:6819/3061361173","cluster_addr":"172.21.15.110:6821/3061361173","heartbeat_back_addr":"172.21.15.110:6825/3061361173","heartbeat_front_addr":"172.21.15.110:6823/3061361173","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6827","nonce":3810786960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6829","nonce":3810786960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6833","nonce":3810786960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6831","nonce":3810786960}]},"public_addr":"172.21.15.110:6827/3810786960","cluster_addr":"172.21.15.110:6829/3810786960","heartbeat_back_addr":"172.21.15.110:6833/3810786960","heartbeat_front_addr":"172.21.15.110:6831/3810786960","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6801","nonce":3499292350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6803","nonce":3499292350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6807","nonce":3499292350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6805","nonce":3499292350}]},"public_addr":"172.21.15.195:6801/3499292350","cluster_addr":"172.21.15.195:6803/3499292350","heartbeat_back_addr":"172.21.15.195:6807/3499292350","heartbeat_front_addr":"172.21.15.195:6805/3499292350","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6809","nonce":4151750662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6811","nonce":4151750662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6815","nonce":4151750662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6813","nonce":4151750662}]},"public_addr":"172.21.15.195:6809/4151750662","cluster_addr":"172.21.15.195:6811/4151750662","heartbeat_back_addr":"172.21.15.195:6815/4151750662","heartbeat_front_addr":"172.21.15.195:6813/4151750662","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6817","nonce":258227795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6819","nonce":258227795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6823","nonce":258227795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6821","nonce":258227795}]},"public_addr":"172.21.15.195:6817/258227795","cluster_addr":"172.21.15.195:6819/258227795","heartbeat_back_addr":"172.21.15.195:6823/258227795","heartbeat_front_addr":"172.21.15.195:6821/258227795","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6825","nonce":284932944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6827","nonce":284932944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6831","nonce":284932944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6829","nonce":284932944}]},"public_addr":"172.21.15.195:6825/284932944","cluster_addr":"172.21.15.195:6827/284932944","heartbeat_back_addr":"172.21.15.195:6831/284932944","heartbeat_front_addr":"172.21.15.195:6829/284932944","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:34:09.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:09 smithi110 ceph-mon[27365]: pgmap v186: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s wr, 6 op/s 2024-09-14T03:34:09.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:09 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2426551377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:09.886 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:09 smithi110 ceph-mon[35374]: pgmap v186: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s wr, 6 op/s 2024-09-14T03:34:09.886 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:09 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2426551377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:09 smithi195 ceph-mon[30695]: pgmap v186: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s wr, 6 op/s 2024-09-14T03:34:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:09 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2426551377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:10.437 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs set cephfs standby_count_wanted 0 2024-09-14T03:34:10.681 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:12.056 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:11 smithi110 ceph-mon[35374]: pgmap v187: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-14T03:34:12.056 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:11 smithi110 ceph-mon[27365]: pgmap v187: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-14T03:34:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:11 smithi195 ceph-mon[30695]: pgmap v187: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-14T03:34:13.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:12 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1927988813' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:34:13.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:12 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1927988813' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:34:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:12 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1927988813' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:34:13.591 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-09-14T03:34:13.591 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs set cephfs allow_standby_replay false 2024-09-14T03:34:13.838 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:13 smithi110 ceph-mon[27365]: pgmap v188: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-14T03:34:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:13 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1927988813' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-14T03:34:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:13 smithi110 ceph-mon[27365]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:13 smithi110 ceph-mon[35374]: pgmap v188: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-14T03:34:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:13 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1927988813' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-14T03:34:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:13 smithi110 ceph-mon[35374]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:14.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:13 smithi195 ceph-mon[30695]: pgmap v188: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-14T03:34:14.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:13 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1927988813' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-14T03:34:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:13 smithi195 ceph-mon[30695]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:16.192 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:15 smithi110 ceph-mon[27365]: pgmap v189: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-14T03:34:16.193 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:15 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4038606817' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:34:16.193 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:15 smithi110 ceph-mon[35374]: pgmap v189: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-14T03:34:16.193 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:15 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/4038606817' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:34:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:15 smithi195 ceph-mon[30695]: pgmap v189: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-14T03:34:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:15 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/4038606817' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:34:16.545 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-09-14T03:34:16.546 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:34:16.789 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:17.172 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:16 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/4038606817' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-14T03:34:17.172 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:16 smithi110 ceph-mon[35374]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:17.173 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:16 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4038606817' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-14T03:34:17.173 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:16 smithi110 ceph-mon[27365]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:17.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:16 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/4038606817' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-14T03:34:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:16 smithi195 ceph-mon[30695]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:34:18.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:17 smithi110 ceph-mon[27365]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-14T03:34:18.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:17 smithi110 ceph-mon[35374]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-14T03:34:18.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:17 smithi195 ceph-mon[30695]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-14T03:34:18.497 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:34:18.498 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24310,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.195:6833/1180864653","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6832","nonce":1180864653},{"type":"v1","addr":"172.21.15.195:6833","nonce":1180864653}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:34:04.160374+0000","modified":"2024-09-14T03:34:15.891707+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24311},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24311":{"gid":24311,"name":"a","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.110:6835/2015946497","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6834","nonce":2015946497},{"type":"v1","addr":"172.21.15.110:6835","nonce":2015946497}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-14T03:34:18.498 INFO:teuthology.orchestra.run.smithi110.stderr:dumped fsmap epoch 7 2024-09-14T03:34:19.068 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:18 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/518355111' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:34:19.068 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:18 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/518355111' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:34:19.093 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:34:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:18 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/518355111' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:34:19.333 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:20.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:19 smithi110 ceph-mon[27365]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 B/s wr, 1 op/s 2024-09-14T03:34:20.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:19 smithi110 ceph-mon[35374]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 B/s wr, 1 op/s 2024-09-14T03:34:20.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:19 smithi195 ceph-mon[30695]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 B/s wr, 1 op/s 2024-09-14T03:34:21.086 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:34:21.086 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":58,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:34:04.160250+0000","last_up_change":"2024-09-14T03:33:34.960002+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:33:57.860054+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:34:01.436541+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6803","nonce":1137454602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6805","nonce":1137454602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6809","nonce":1137454602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6807","nonce":1137454602}]},"public_addr":"172.21.15.110:6803/1137454602","cluster_addr":"172.21.15.110:6805/1137454602","heartbeat_back_addr":"172.21.15.110:6809/1137454602","heartbeat_front_addr":"172.21.15.110:6807/1137454602","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6811","nonce":3479666487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6813","nonce":3479666487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6817","nonce":3479666487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6815","nonce":3479666487}]},"public_addr":"172.21.15.110:6811/3479666487","cluster_addr":"172.21.15.110:6813/3479666487","heartbeat_back_addr":"172.21.15.110:6817/3479666487","heartbeat_front_addr":"172.21.15.110:6815/3479666487","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6819","nonce":3061361173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6821","nonce":3061361173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6825","nonce":3061361173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6823","nonce":3061361173}]},"public_addr":"172.21.15.110:6819/3061361173","cluster_addr":"172.21.15.110:6821/3061361173","heartbeat_back_addr":"172.21.15.110:6825/3061361173","heartbeat_front_addr":"172.21.15.110:6823/3061361173","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6827","nonce":3810786960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6829","nonce":3810786960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6833","nonce":3810786960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6831","nonce":3810786960}]},"public_addr":"172.21.15.110:6827/3810786960","cluster_addr":"172.21.15.110:6829/3810786960","heartbeat_back_addr":"172.21.15.110:6833/3810786960","heartbeat_front_addr":"172.21.15.110:6831/3810786960","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6801","nonce":3499292350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6803","nonce":3499292350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6807","nonce":3499292350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6805","nonce":3499292350}]},"public_addr":"172.21.15.195:6801/3499292350","cluster_addr":"172.21.15.195:6803/3499292350","heartbeat_back_addr":"172.21.15.195:6807/3499292350","heartbeat_front_addr":"172.21.15.195:6805/3499292350","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6809","nonce":4151750662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6811","nonce":4151750662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6815","nonce":4151750662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6813","nonce":4151750662}]},"public_addr":"172.21.15.195:6809/4151750662","cluster_addr":"172.21.15.195:6811/4151750662","heartbeat_back_addr":"172.21.15.195:6815/4151750662","heartbeat_front_addr":"172.21.15.195:6813/4151750662","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6817","nonce":258227795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6819","nonce":258227795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6823","nonce":258227795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6821","nonce":258227795}]},"public_addr":"172.21.15.195:6817/258227795","cluster_addr":"172.21.15.195:6819/258227795","heartbeat_back_addr":"172.21.15.195:6823/258227795","heartbeat_front_addr":"172.21.15.195:6821/258227795","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6825","nonce":284932944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6827","nonce":284932944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6831","nonce":284932944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6829","nonce":284932944}]},"public_addr":"172.21.15.195:6825/284932944","cluster_addr":"172.21.15.195:6827/284932944","heartbeat_back_addr":"172.21.15.195:6831/284932944","heartbeat_front_addr":"172.21.15.195:6829/284932944","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:34:21.634 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T03:34:21.634 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:34:21.875 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:22.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:21 smithi110 ceph-mon[35374]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:34:22.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:21 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1344905814' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:22.161 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:21 smithi110 ceph-mon[27365]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:34:22.161 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:21 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1344905814' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:21 smithi195 ceph-mon[30695]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:34:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:21 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1344905814' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:34:23.802 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:34:23.804 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2024-09-14T03:34:24.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:23 smithi110 ceph-mon[35374]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:24.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:23 smithi110 ceph-mon[27365]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:24.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:23 smithi195 ceph-mon[30695]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:24.500 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":193,"stamp":"2024-09-14T03:34:22.832408+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385188,"kb_used_data":6292,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576028,"statfs":{"total":767960285184,"available":765517852672,"internally_reserved":0,"allocated":6443008,"data_stored":3701195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002929"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508680+0000","last_change":"2024-09-14T03:34:02.519367+0000","last_active":"2024-09-14T03:34:03.508680+0000","last_peered":"2024-09-14T03:34:03.508680+0000","last_clean":"2024-09-14T03:34:03.508680+0000","last_became_active":"2024-09-14T03:34:02.518530+0000","last_became_peered":"2024-09-14T03:34:02.518530+0000","last_unstale":"2024-09-14T03:34:03.508680+0000","last_undegraded":"2024-09-14T03:34:03.508680+0000","last_fullsized":"2024-09-14T03:34:03.508680+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:02:36.205002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479156+0000","last_change":"2024-09-14T03:34:02.515212+0000","last_active":"2024-09-14T03:34:04.479156+0000","last_peered":"2024-09-14T03:34:04.479156+0000","last_clean":"2024-09-14T03:34:04.479156+0000","last_became_active":"2024-09-14T03:34:02.514951+0000","last_became_peered":"2024-09-14T03:34:02.514951+0000","last_unstale":"2024-09-14T03:34:04.479156+0000","last_undegraded":"2024-09-14T03:34:04.479156+0000","last_fullsized":"2024-09-14T03:34:04.479156+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:30.742131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508290+0000","last_change":"2024-09-14T03:34:02.504777+0000","last_active":"2024-09-14T03:34:03.508290+0000","last_peered":"2024-09-14T03:34:03.508290+0000","last_clean":"2024-09-14T03:34:03.508290+0000","last_became_active":"2024-09-14T03:34:02.504550+0000","last_became_peered":"2024-09-14T03:34:02.504550+0000","last_unstale":"2024-09-14T03:34:03.508290+0000","last_undegraded":"2024-09-14T03:34:03.508290+0000","last_fullsized":"2024-09-14T03:34:03.508290+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:59.864492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479137+0000","last_change":"2024-09-14T03:34:02.521444+0000","last_active":"2024-09-14T03:34:04.479137+0000","last_peered":"2024-09-14T03:34:04.479137+0000","last_clean":"2024-09-14T03:34:04.479137+0000","last_became_active":"2024-09-14T03:34:02.521149+0000","last_became_peered":"2024-09-14T03:34:02.521149+0000","last_unstale":"2024-09-14T03:34:04.479137+0000","last_undegraded":"2024-09-14T03:34:04.479137+0000","last_fullsized":"2024-09-14T03:34:04.479137+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:10.004625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508126+0000","last_change":"2024-09-14T03:34:02.513133+0000","last_active":"2024-09-14T03:34:03.508126+0000","last_peered":"2024-09-14T03:34:03.508126+0000","last_clean":"2024-09-14T03:34:03.508126+0000","last_became_active":"2024-09-14T03:34:02.512580+0000","last_became_peered":"2024-09-14T03:34:02.512580+0000","last_unstale":"2024-09-14T03:34:03.508126+0000","last_undegraded":"2024-09-14T03:34:03.508126+0000","last_fullsized":"2024-09-14T03:34:03.508126+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:03.077106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479529+0000","last_change":"2024-09-14T03:34:02.508230+0000","last_active":"2024-09-14T03:34:04.479529+0000","last_peered":"2024-09-14T03:34:04.479529+0000","last_clean":"2024-09-14T03:34:04.479529+0000","last_became_active":"2024-09-14T03:34:02.507921+0000","last_became_peered":"2024-09-14T03:34:02.507921+0000","last_unstale":"2024-09-14T03:34:04.479529+0000","last_undegraded":"2024-09-14T03:34:04.479529+0000","last_fullsized":"2024-09-14T03:34:04.479529+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:37.418870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507913+0000","last_change":"2024-09-14T03:34:02.515319+0000","last_active":"2024-09-14T03:34:03.507913+0000","last_peered":"2024-09-14T03:34:03.507913+0000","last_clean":"2024-09-14T03:34:03.507913+0000","last_became_active":"2024-09-14T03:34:02.515022+0000","last_became_peered":"2024-09-14T03:34:02.515022+0000","last_unstale":"2024-09-14T03:34:03.507913+0000","last_undegraded":"2024-09-14T03:34:03.507913+0000","last_fullsized":"2024-09-14T03:34:03.507913+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:51:51.089278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478715+0000","last_change":"2024-09-14T03:34:02.522349+0000","last_active":"2024-09-14T03:34:04.478715+0000","last_peered":"2024-09-14T03:34:04.478715+0000","last_clean":"2024-09-14T03:34:04.478715+0000","last_became_active":"2024-09-14T03:34:02.522063+0000","last_became_peered":"2024-09-14T03:34:02.522063+0000","last_unstale":"2024-09-14T03:34:04.478715+0000","last_undegraded":"2024-09-14T03:34:04.478715+0000","last_fullsized":"2024-09-14T03:34:04.478715+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:15.378220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508868+0000","last_change":"2024-09-14T03:34:02.512031+0000","last_active":"2024-09-14T03:34:03.508868+0000","last_peered":"2024-09-14T03:34:03.508868+0000","last_clean":"2024-09-14T03:34:03.508868+0000","last_became_active":"2024-09-14T03:34:02.511364+0000","last_became_peered":"2024-09-14T03:34:02.511364+0000","last_unstale":"2024-09-14T03:34:03.508868+0000","last_undegraded":"2024-09-14T03:34:03.508868+0000","last_fullsized":"2024-09-14T03:34:03.508868+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:54:15.711934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547595+0000","last_change":"2024-09-14T03:34:02.518932+0000","last_active":"2024-09-14T03:34:03.547595+0000","last_peered":"2024-09-14T03:34:03.547595+0000","last_clean":"2024-09-14T03:34:03.547595+0000","last_became_active":"2024-09-14T03:34:02.518575+0000","last_became_peered":"2024-09-14T03:34:02.518575+0000","last_unstale":"2024-09-14T03:34:03.547595+0000","last_undegraded":"2024-09-14T03:34:03.547595+0000","last_fullsized":"2024-09-14T03:34:03.547595+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:01:11.929183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547443+0000","last_change":"2024-09-14T03:34:02.517977+0000","last_active":"2024-09-14T03:34:03.547443+0000","last_peered":"2024-09-14T03:34:03.547443+0000","last_clean":"2024-09-14T03:34:03.547443+0000","last_became_active":"2024-09-14T03:34:02.517385+0000","last_became_peered":"2024-09-14T03:34:02.517385+0000","last_unstale":"2024-09-14T03:34:03.547443+0000","last_undegraded":"2024-09-14T03:34:03.547443+0000","last_fullsized":"2024-09-14T03:34:03.547443+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:50.900533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479601+0000","last_change":"2024-09-14T03:34:02.510949+0000","last_active":"2024-09-14T03:34:04.479601+0000","last_peered":"2024-09-14T03:34:04.479601+0000","last_clean":"2024-09-14T03:34:04.479601+0000","last_became_active":"2024-09-14T03:34:02.510519+0000","last_became_peered":"2024-09-14T03:34:02.510519+0000","last_unstale":"2024-09-14T03:34:04.479601+0000","last_undegraded":"2024-09-14T03:34:04.479601+0000","last_fullsized":"2024-09-14T03:34:04.479601+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:06:16.860908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478947+0000","last_change":"2024-09-14T03:34:02.512463+0000","last_active":"2024-09-14T03:34:04.478947+0000","last_peered":"2024-09-14T03:34:04.478947+0000","last_clean":"2024-09-14T03:34:04.478947+0000","last_became_active":"2024-09-14T03:34:02.512201+0000","last_became_peered":"2024-09-14T03:34:02.512201+0000","last_unstale":"2024-09-14T03:34:04.478947+0000","last_undegraded":"2024-09-14T03:34:04.478947+0000","last_fullsized":"2024-09-14T03:34:04.478947+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:46.716482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508412+0000","last_change":"2024-09-14T03:34:02.513215+0000","last_active":"2024-09-14T03:34:03.508412+0000","last_peered":"2024-09-14T03:34:03.508412+0000","last_clean":"2024-09-14T03:34:03.508412+0000","last_became_active":"2024-09-14T03:34:02.512773+0000","last_became_peered":"2024-09-14T03:34:02.512773+0000","last_unstale":"2024-09-14T03:34:03.508412+0000","last_undegraded":"2024-09-14T03:34:03.508412+0000","last_fullsized":"2024-09-14T03:34:03.508412+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:27:04.454480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478731+0000","last_change":"2024-09-14T03:34:02.503785+0000","last_active":"2024-09-14T03:34:04.478731+0000","last_peered":"2024-09-14T03:34:04.478731+0000","last_clean":"2024-09-14T03:34:04.478731+0000","last_became_active":"2024-09-14T03:34:02.503212+0000","last_became_peered":"2024-09-14T03:34:02.503212+0000","last_unstale":"2024-09-14T03:34:04.478731+0000","last_undegraded":"2024-09-14T03:34:04.478731+0000","last_fullsized":"2024-09-14T03:34:04.478731+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:15.408242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478728+0000","last_change":"2024-09-14T03:34:02.510927+0000","last_active":"2024-09-14T03:34:04.478728+0000","last_peered":"2024-09-14T03:34:04.478728+0000","last_clean":"2024-09-14T03:34:04.478728+0000","last_became_active":"2024-09-14T03:34:02.510575+0000","last_became_peered":"2024-09-14T03:34:02.510575+0000","last_unstale":"2024-09-14T03:34:04.478728+0000","last_undegraded":"2024-09-14T03:34:04.478728+0000","last_fullsized":"2024-09-14T03:34:04.478728+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:14.173783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479228+0000","last_change":"2024-09-14T03:34:02.507186+0000","last_active":"2024-09-14T03:34:04.479228+0000","last_peered":"2024-09-14T03:34:04.479228+0000","last_clean":"2024-09-14T03:34:04.479228+0000","last_became_active":"2024-09-14T03:34:02.506907+0000","last_became_peered":"2024-09-14T03:34:02.506907+0000","last_unstale":"2024-09-14T03:34:04.479228+0000","last_undegraded":"2024-09-14T03:34:04.479228+0000","last_fullsized":"2024-09-14T03:34:04.479228+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:10:47.570717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508073+0000","last_change":"2024-09-14T03:34:02.519104+0000","last_active":"2024-09-14T03:34:03.508073+0000","last_peered":"2024-09-14T03:34:03.508073+0000","last_clean":"2024-09-14T03:34:03.508073+0000","last_became_active":"2024-09-14T03:34:02.518795+0000","last_became_peered":"2024-09-14T03:34:02.518795+0000","last_unstale":"2024-09-14T03:34:03.508073+0000","last_undegraded":"2024-09-14T03:34:03.508073+0000","last_fullsized":"2024-09-14T03:34:03.508073+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:34:49.356603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508498+0000","last_change":"2024-09-14T03:34:02.521848+0000","last_active":"2024-09-14T03:34:03.508498+0000","last_peered":"2024-09-14T03:34:03.508498+0000","last_clean":"2024-09-14T03:34:03.508498+0000","last_became_active":"2024-09-14T03:34:02.521559+0000","last_became_peered":"2024-09-14T03:34:02.521559+0000","last_unstale":"2024-09-14T03:34:03.508498+0000","last_undegraded":"2024-09-14T03:34:03.508498+0000","last_fullsized":"2024-09-14T03:34:03.508498+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:21:09.742073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547522+0000","last_change":"2024-09-14T03:34:02.515094+0000","last_active":"2024-09-14T03:34:03.547522+0000","last_peered":"2024-09-14T03:34:03.547522+0000","last_clean":"2024-09-14T03:34:03.547522+0000","last_became_active":"2024-09-14T03:34:02.514291+0000","last_became_peered":"2024-09-14T03:34:02.514291+0000","last_unstale":"2024-09-14T03:34:03.547522+0000","last_undegraded":"2024-09-14T03:34:03.547522+0000","last_fullsized":"2024-09-14T03:34:03.547522+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:39:52.842240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508310+0000","last_change":"2024-09-14T03:34:02.510758+0000","last_active":"2024-09-14T03:34:03.508310+0000","last_peered":"2024-09-14T03:34:03.508310+0000","last_clean":"2024-09-14T03:34:03.508310+0000","last_became_active":"2024-09-14T03:34:02.510485+0000","last_became_peered":"2024-09-14T03:34:02.510485+0000","last_unstale":"2024-09-14T03:34:03.508310+0000","last_undegraded":"2024-09-14T03:34:03.508310+0000","last_fullsized":"2024-09-14T03:34:03.508310+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:26:25.102759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479538+0000","last_change":"2024-09-14T03:34:02.511696+0000","last_active":"2024-09-14T03:34:04.479538+0000","last_peered":"2024-09-14T03:34:04.479538+0000","last_clean":"2024-09-14T03:34:04.479538+0000","last_became_active":"2024-09-14T03:34:02.511289+0000","last_became_peered":"2024-09-14T03:34:02.511289+0000","last_unstale":"2024-09-14T03:34:04.479538+0000","last_undegraded":"2024-09-14T03:34:04.479538+0000","last_fullsized":"2024-09-14T03:34:04.479538+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:02:27.657439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478791+0000","last_change":"2024-09-14T03:34:02.517564+0000","last_active":"2024-09-14T03:34:04.478791+0000","last_peered":"2024-09-14T03:34:04.478791+0000","last_clean":"2024-09-14T03:34:04.478791+0000","last_became_active":"2024-09-14T03:34:02.517314+0000","last_became_peered":"2024-09-14T03:34:02.517314+0000","last_unstale":"2024-09-14T03:34:04.478791+0000","last_undegraded":"2024-09-14T03:34:04.478791+0000","last_fullsized":"2024-09-14T03:34:04.478791+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:08.360046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478817+0000","last_change":"2024-09-14T03:34:02.513865+0000","last_active":"2024-09-14T03:34:04.478817+0000","last_peered":"2024-09-14T03:34:04.478817+0000","last_clean":"2024-09-14T03:34:04.478817+0000","last_became_active":"2024-09-14T03:34:02.513008+0000","last_became_peered":"2024-09-14T03:34:02.513008+0000","last_unstale":"2024-09-14T03:34:04.478817+0000","last_undegraded":"2024-09-14T03:34:04.478817+0000","last_fullsized":"2024-09-14T03:34:04.478817+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:18:37.581596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479390+0000","last_change":"2024-09-14T03:34:02.517481+0000","last_active":"2024-09-14T03:34:04.479390+0000","last_peered":"2024-09-14T03:34:04.479390+0000","last_clean":"2024-09-14T03:34:04.479390+0000","last_became_active":"2024-09-14T03:34:02.517252+0000","last_became_peered":"2024-09-14T03:34:02.517252+0000","last_unstale":"2024-09-14T03:34:04.479390+0000","last_undegraded":"2024-09-14T03:34:04.479390+0000","last_fullsized":"2024-09-14T03:34:04.479390+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:29:15.209238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547669+0000","last_change":"2024-09-14T03:34:02.521201+0000","last_active":"2024-09-14T03:34:03.547669+0000","last_peered":"2024-09-14T03:34:03.547669+0000","last_clean":"2024-09-14T03:34:03.547669+0000","last_became_active":"2024-09-14T03:34:02.520764+0000","last_became_peered":"2024-09-14T03:34:02.520764+0000","last_unstale":"2024-09-14T03:34:03.547669+0000","last_undegraded":"2024-09-14T03:34:03.547669+0000","last_fullsized":"2024-09-14T03:34:03.547669+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:39:40.688146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479030+0000","last_change":"2024-09-14T03:34:02.504156+0000","last_active":"2024-09-14T03:34:04.479030+0000","last_peered":"2024-09-14T03:34:04.479030+0000","last_clean":"2024-09-14T03:34:04.479030+0000","last_became_active":"2024-09-14T03:34:02.503906+0000","last_became_peered":"2024-09-14T03:34:02.503906+0000","last_unstale":"2024-09-14T03:34:04.479030+0000","last_undegraded":"2024-09-14T03:34:04.479030+0000","last_fullsized":"2024-09-14T03:34:04.479030+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:33:57.350618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479315+0000","last_change":"2024-09-14T03:34:02.518555+0000","last_active":"2024-09-14T03:34:04.479315+0000","last_peered":"2024-09-14T03:34:04.479315+0000","last_clean":"2024-09-14T03:34:04.479315+0000","last_became_active":"2024-09-14T03:34:02.517920+0000","last_became_peered":"2024-09-14T03:34:02.517920+0000","last_unstale":"2024-09-14T03:34:04.479315+0000","last_undegraded":"2024-09-14T03:34:04.479315+0000","last_fullsized":"2024-09-14T03:34:04.479315+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:54.031933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508385+0000","last_change":"2024-09-14T03:34:02.515181+0000","last_active":"2024-09-14T03:34:03.508385+0000","last_peered":"2024-09-14T03:34:03.508385+0000","last_clean":"2024-09-14T03:34:03.508385+0000","last_became_active":"2024-09-14T03:34:02.514649+0000","last_became_peered":"2024-09-14T03:34:02.514649+0000","last_unstale":"2024-09-14T03:34:03.508385+0000","last_undegraded":"2024-09-14T03:34:03.508385+0000","last_fullsized":"2024-09-14T03:34:03.508385+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:05.770205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479663+0000","last_change":"2024-09-14T03:34:02.513294+0000","last_active":"2024-09-14T03:34:04.479663+0000","last_peered":"2024-09-14T03:34:04.479663+0000","last_clean":"2024-09-14T03:34:04.479663+0000","last_became_active":"2024-09-14T03:34:02.512998+0000","last_became_peered":"2024-09-14T03:34:02.512998+0000","last_unstale":"2024-09-14T03:34:04.479663+0000","last_undegraded":"2024-09-14T03:34:04.479663+0000","last_fullsized":"2024-09-14T03:34:04.479663+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:54.262830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507418+0000","last_change":"2024-09-14T03:34:02.512389+0000","last_active":"2024-09-14T03:34:03.507418+0000","last_peered":"2024-09-14T03:34:03.507418+0000","last_clean":"2024-09-14T03:34:03.507418+0000","last_became_active":"2024-09-14T03:34:02.511955+0000","last_became_peered":"2024-09-14T03:34:02.511955+0000","last_unstale":"2024-09-14T03:34:03.507418+0000","last_undegraded":"2024-09-14T03:34:03.507418+0000","last_fullsized":"2024-09-14T03:34:03.507418+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:30:31.454464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547061+0000","last_change":"2024-09-14T03:34:02.520047+0000","last_active":"2024-09-14T03:34:03.547061+0000","last_peered":"2024-09-14T03:34:03.547061+0000","last_clean":"2024-09-14T03:34:03.547061+0000","last_became_active":"2024-09-14T03:34:02.519764+0000","last_became_peered":"2024-09-14T03:34:02.519764+0000","last_unstale":"2024-09-14T03:34:03.547061+0000","last_undegraded":"2024-09-14T03:34:03.547061+0000","last_fullsized":"2024-09-14T03:34:03.547061+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:47:55.233617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478695+0000","last_change":"2024-09-14T03:34:02.515661+0000","last_active":"2024-09-14T03:34:04.478695+0000","last_peered":"2024-09-14T03:34:04.478695+0000","last_clean":"2024-09-14T03:34:04.478695+0000","last_became_active":"2024-09-14T03:34:02.515374+0000","last_became_peered":"2024-09-14T03:34:02.515374+0000","last_unstale":"2024-09-14T03:34:04.478695+0000","last_undegraded":"2024-09-14T03:34:04.478695+0000","last_fullsized":"2024-09-14T03:34:04.478695+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:06.066379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.516531+0000","last_change":"2024-09-14T03:33:59.400168+0000","last_active":"2024-09-14T03:34:04.516531+0000","last_peered":"2024-09-14T03:34:04.516531+0000","last_clean":"2024-09-14T03:34:04.516531+0000","last_became_active":"2024-09-14T03:33:59.399904+0000","last_became_peered":"2024-09-14T03:33:59.399904+0000","last_unstale":"2024-09-14T03:34:04.516531+0000","last_undegraded":"2024-09-14T03:34:04.516531+0000","last_fullsized":"2024-09-14T03:34:04.516531+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.387300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508444+0000","last_change":"2024-09-14T03:34:02.514049+0000","last_active":"2024-09-14T03:34:03.508444+0000","last_peered":"2024-09-14T03:34:03.508444+0000","last_clean":"2024-09-14T03:34:03.508444+0000","last_became_active":"2024-09-14T03:34:02.513309+0000","last_became_peered":"2024-09-14T03:34:02.513309+0000","last_unstale":"2024-09-14T03:34:03.508444+0000","last_undegraded":"2024-09-14T03:34:03.508444+0000","last_fullsized":"2024-09-14T03:34:03.508444+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:14.127299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.514634+0000","last_change":"2024-09-14T03:33:59.406006+0000","last_active":"2024-09-14T03:34:04.514634+0000","last_peered":"2024-09-14T03:34:04.514634+0000","last_clean":"2024-09-14T03:34:04.514634+0000","last_became_active":"2024-09-14T03:33:59.405514+0000","last_became_peered":"2024-09-14T03:33:59.405514+0000","last_unstale":"2024-09-14T03:34:04.514634+0000","last_undegraded":"2024-09-14T03:34:04.514634+0000","last_fullsized":"2024-09-14T03:34:04.514634+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:23:17.405355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478948+0000","last_change":"2024-09-14T03:33:59.399102+0000","last_active":"2024-09-14T03:34:04.478948+0000","last_peered":"2024-09-14T03:34:04.478948+0000","last_clean":"2024-09-14T03:34:04.478948+0000","last_became_active":"2024-09-14T03:33:59.398766+0000","last_became_peered":"2024-09-14T03:33:59.398766+0000","last_unstale":"2024-09-14T03:34:04.478948+0000","last_undegraded":"2024-09-14T03:34:04.478948+0000","last_fullsized":"2024-09-14T03:34:04.478948+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:08:01.084059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478803+0000","last_change":"2024-09-14T03:34:02.520106+0000","last_active":"2024-09-14T03:34:04.478803+0000","last_peered":"2024-09-14T03:34:04.478803+0000","last_clean":"2024-09-14T03:34:04.478803+0000","last_became_active":"2024-09-14T03:34:02.519825+0000","last_became_peered":"2024-09-14T03:34:02.519825+0000","last_unstale":"2024-09-14T03:34:04.478803+0000","last_undegraded":"2024-09-14T03:34:04.478803+0000","last_fullsized":"2024-09-14T03:34:04.478803+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:44.762587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478888+0000","last_change":"2024-09-14T03:33:59.406587+0000","last_active":"2024-09-14T03:34:04.478888+0000","last_peered":"2024-09-14T03:34:04.478888+0000","last_clean":"2024-09-14T03:34:04.478888+0000","last_became_active":"2024-09-14T03:33:59.405828+0000","last_became_peered":"2024-09-14T03:33:59.405828+0000","last_unstale":"2024-09-14T03:34:04.478888+0000","last_undegraded":"2024-09-14T03:34:04.478888+0000","last_fullsized":"2024-09-14T03:34:04.478888+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:57.857900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478585+0000","last_change":"2024-09-14T03:34:02.507004+0000","last_active":"2024-09-14T03:34:04.478585+0000","last_peered":"2024-09-14T03:34:04.478585+0000","last_clean":"2024-09-14T03:34:04.478585+0000","last_became_active":"2024-09-14T03:34:02.506311+0000","last_became_peered":"2024-09-14T03:34:02.506311+0000","last_unstale":"2024-09-14T03:34:04.478585+0000","last_undegraded":"2024-09-14T03:34:04.478585+0000","last_fullsized":"2024-09-14T03:34:04.478585+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:33.038366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547596+0000","last_change":"2024-09-14T03:33:59.402116+0000","last_active":"2024-09-14T03:34:03.547596+0000","last_peered":"2024-09-14T03:34:03.547596+0000","last_clean":"2024-09-14T03:34:03.547596+0000","last_became_active":"2024-09-14T03:33:59.400274+0000","last_became_peered":"2024-09-14T03:33:59.400274+0000","last_unstale":"2024-09-14T03:34:03.547596+0000","last_undegraded":"2024-09-14T03:34:03.547596+0000","last_fullsized":"2024-09-14T03:34:03.547596+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:46.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478905+0000","last_change":"2024-09-14T03:34:02.520187+0000","last_active":"2024-09-14T03:34:04.478905+0000","last_peered":"2024-09-14T03:34:04.478905+0000","last_clean":"2024-09-14T03:34:04.478905+0000","last_became_active":"2024-09-14T03:34:02.519909+0000","last_became_peered":"2024-09-14T03:34:02.519909+0000","last_unstale":"2024-09-14T03:34:04.478905+0000","last_undegraded":"2024-09-14T03:34:04.478905+0000","last_fullsized":"2024-09-14T03:34:04.478905+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:32.588302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479053+0000","last_change":"2024-09-14T03:33:59.402069+0000","last_active":"2024-09-14T03:34:04.479053+0000","last_peered":"2024-09-14T03:34:04.479053+0000","last_clean":"2024-09-14T03:34:04.479053+0000","last_became_active":"2024-09-14T03:33:59.401698+0000","last_became_peered":"2024-09-14T03:33:59.401698+0000","last_unstale":"2024-09-14T03:34:04.479053+0000","last_undegraded":"2024-09-14T03:34:04.479053+0000","last_fullsized":"2024-09-14T03:34:04.479053+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:16:48.483610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508760+0000","last_change":"2024-09-14T03:34:02.515495+0000","last_active":"2024-09-14T03:34:03.508760+0000","last_peered":"2024-09-14T03:34:03.508760+0000","last_clean":"2024-09-14T03:34:03.508760+0000","last_became_active":"2024-09-14T03:34:02.515256+0000","last_became_peered":"2024-09-14T03:34:02.515256+0000","last_unstale":"2024-09-14T03:34:03.508760+0000","last_undegraded":"2024-09-14T03:34:03.508760+0000","last_fullsized":"2024-09-14T03:34:03.508760+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:02.392960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.520648+0000","last_change":"2024-09-14T03:33:59.398566+0000","last_active":"2024-09-14T03:34:04.520648+0000","last_peered":"2024-09-14T03:34:04.520648+0000","last_clean":"2024-09-14T03:34:04.520648+0000","last_became_active":"2024-09-14T03:33:59.398281+0000","last_became_peered":"2024-09-14T03:33:59.398281+0000","last_unstale":"2024-09-14T03:34:04.520648+0000","last_undegraded":"2024-09-14T03:34:04.520648+0000","last_fullsized":"2024-09-14T03:34:04.520648+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:42.503990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479665+0000","last_change":"2024-09-14T03:34:02.510787+0000","last_active":"2024-09-14T03:34:04.479665+0000","last_peered":"2024-09-14T03:34:04.479665+0000","last_clean":"2024-09-14T03:34:04.479665+0000","last_became_active":"2024-09-14T03:34:02.510117+0000","last_became_peered":"2024-09-14T03:34:02.510117+0000","last_unstale":"2024-09-14T03:34:04.479665+0000","last_undegraded":"2024-09-14T03:34:04.479665+0000","last_fullsized":"2024-09-14T03:34:04.479665+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:13:53.616471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"58'6","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.522253+0000","last_change":"2024-09-14T03:33:59.401007+0000","last_active":"2024-09-14T03:34:04.522253+0000","last_peered":"2024-09-14T03:34:04.522253+0000","last_clean":"2024-09-14T03:34:04.522253+0000","last_became_active":"2024-09-14T03:33:59.400453+0000","last_became_peered":"2024-09-14T03:33:59.400453+0000","last_unstale":"2024-09-14T03:34:04.522253+0000","last_undegraded":"2024-09-14T03:34:04.522253+0000","last_fullsized":"2024-09-14T03:34:04.522253+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:32.633267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508331+0000","last_change":"2024-09-14T03:34:02.511714+0000","last_active":"2024-09-14T03:34:03.508331+0000","last_peered":"2024-09-14T03:34:03.508331+0000","last_clean":"2024-09-14T03:34:03.508331+0000","last_became_active":"2024-09-14T03:34:02.511328+0000","last_became_peered":"2024-09-14T03:34:02.511328+0000","last_unstale":"2024-09-14T03:34:03.508331+0000","last_undegraded":"2024-09-14T03:34:03.508331+0000","last_fullsized":"2024-09-14T03:34:03.508331+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:00:06.445335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.523143+0000","last_change":"2024-09-14T03:33:59.396984+0000","last_active":"2024-09-14T03:34:04.523143+0000","last_peered":"2024-09-14T03:34:04.523143+0000","last_clean":"2024-09-14T03:34:04.523143+0000","last_became_active":"2024-09-14T03:33:59.395835+0000","last_became_peered":"2024-09-14T03:33:59.395835+0000","last_unstale":"2024-09-14T03:34:04.523143+0000","last_undegraded":"2024-09-14T03:34:04.523143+0000","last_fullsized":"2024-09-14T03:34:04.523143+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:57.889158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508491+0000","last_change":"2024-09-14T03:34:02.521192+0000","last_active":"2024-09-14T03:34:03.508491+0000","last_peered":"2024-09-14T03:34:03.508491+0000","last_clean":"2024-09-14T03:34:03.508491+0000","last_became_active":"2024-09-14T03:34:02.520704+0000","last_became_peered":"2024-09-14T03:34:02.520704+0000","last_unstale":"2024-09-14T03:34:03.508491+0000","last_undegraded":"2024-09-14T03:34:03.508491+0000","last_fullsized":"2024-09-14T03:34:03.508491+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:37:13.469199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479662+0000","last_change":"2024-09-14T03:33:59.406737+0000","last_active":"2024-09-14T03:34:04.479662+0000","last_peered":"2024-09-14T03:34:04.479662+0000","last_clean":"2024-09-14T03:34:04.479662+0000","last_became_active":"2024-09-14T03:33:59.405882+0000","last_became_peered":"2024-09-14T03:33:59.405882+0000","last_unstale":"2024-09-14T03:34:04.479662+0000","last_undegraded":"2024-09-14T03:34:04.479662+0000","last_fullsized":"2024-09-14T03:34:04.479662+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:58.379549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508116+0000","last_change":"2024-09-14T03:34:02.517299+0000","last_active":"2024-09-14T03:34:03.508116+0000","last_peered":"2024-09-14T03:34:03.508116+0000","last_clean":"2024-09-14T03:34:03.508116+0000","last_became_active":"2024-09-14T03:34:02.516007+0000","last_became_peered":"2024-09-14T03:34:02.516007+0000","last_unstale":"2024-09-14T03:34:03.508116+0000","last_undegraded":"2024-09-14T03:34:03.508116+0000","last_fullsized":"2024-09-14T03:34:03.508116+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:49.173576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547149+0000","last_change":"2024-09-14T03:33:59.401544+0000","last_active":"2024-09-14T03:34:03.547149+0000","last_peered":"2024-09-14T03:34:03.547149+0000","last_clean":"2024-09-14T03:34:03.547149+0000","last_became_active":"2024-09-14T03:33:59.400073+0000","last_became_peered":"2024-09-14T03:33:59.400073+0000","last_unstale":"2024-09-14T03:34:03.547149+0000","last_undegraded":"2024-09-14T03:34:03.547149+0000","last_fullsized":"2024-09-14T03:34:03.547149+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:24.935055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547180+0000","last_change":"2024-09-14T03:34:02.516939+0000","last_active":"2024-09-14T03:34:03.547180+0000","last_peered":"2024-09-14T03:34:03.547180+0000","last_clean":"2024-09-14T03:34:03.547180+0000","last_became_active":"2024-09-14T03:34:02.516657+0000","last_became_peered":"2024-09-14T03:34:02.516657+0000","last_unstale":"2024-09-14T03:34:03.547180+0000","last_undegraded":"2024-09-14T03:34:03.547180+0000","last_fullsized":"2024-09-14T03:34:03.547180+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:17.423269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.523860+0000","last_change":"2024-09-14T03:33:59.400570+0000","last_active":"2024-09-14T03:34:04.523860+0000","last_peered":"2024-09-14T03:34:04.523860+0000","last_clean":"2024-09-14T03:34:04.523860+0000","last_became_active":"2024-09-14T03:33:59.400323+0000","last_became_peered":"2024-09-14T03:33:59.400323+0000","last_unstale":"2024-09-14T03:34:04.523860+0000","last_undegraded":"2024-09-14T03:34:04.523860+0000","last_fullsized":"2024-09-14T03:34:04.523860+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:39:46.484991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507971+0000","last_change":"2024-09-14T03:34:02.521031+0000","last_active":"2024-09-14T03:34:03.507971+0000","last_peered":"2024-09-14T03:34:03.507971+0000","last_clean":"2024-09-14T03:34:03.507971+0000","last_became_active":"2024-09-14T03:34:02.519973+0000","last_became_peered":"2024-09-14T03:34:02.519973+0000","last_unstale":"2024-09-14T03:34:03.507971+0000","last_undegraded":"2024-09-14T03:34:03.507971+0000","last_fullsized":"2024-09-14T03:34:03.507971+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:20.217955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.510003+0000","last_change":"2024-09-14T03:33:59.399709+0000","last_active":"2024-09-14T03:34:04.510003+0000","last_peered":"2024-09-14T03:34:04.510003+0000","last_clean":"2024-09-14T03:34:04.510003+0000","last_became_active":"2024-09-14T03:33:59.398533+0000","last_became_peered":"2024-09-14T03:33:59.398533+0000","last_unstale":"2024-09-14T03:34:04.510003+0000","last_undegraded":"2024-09-14T03:34:04.510003+0000","last_fullsized":"2024-09-14T03:34:04.510003+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:12:43.525175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508711+0000","last_change":"2024-09-14T03:34:02.520629+0000","last_active":"2024-09-14T03:34:03.508711+0000","last_peered":"2024-09-14T03:34:03.508711+0000","last_clean":"2024-09-14T03:34:03.508711+0000","last_became_active":"2024-09-14T03:34:02.520365+0000","last_became_peered":"2024-09-14T03:34:02.520365+0000","last_unstale":"2024-09-14T03:34:03.508711+0000","last_undegraded":"2024-09-14T03:34:03.508711+0000","last_fullsized":"2024-09-14T03:34:03.508711+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:29.519783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.523660+0000","last_change":"2024-09-14T03:33:59.404858+0000","last_active":"2024-09-14T03:34:04.523660+0000","last_peered":"2024-09-14T03:34:04.523660+0000","last_clean":"2024-09-14T03:34:04.523660+0000","last_became_active":"2024-09-14T03:33:59.404261+0000","last_became_peered":"2024-09-14T03:33:59.404261+0000","last_unstale":"2024-09-14T03:34:04.523660+0000","last_undegraded":"2024-09-14T03:34:04.523660+0000","last_fullsized":"2024-09-14T03:34:04.523660+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:24:38.834576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508776+0000","last_change":"2024-09-14T03:34:02.516816+0000","last_active":"2024-09-14T03:34:03.508776+0000","last_peered":"2024-09-14T03:34:03.508776+0000","last_clean":"2024-09-14T03:34:03.508776+0000","last_became_active":"2024-09-14T03:34:02.516527+0000","last_became_peered":"2024-09-14T03:34:02.516527+0000","last_unstale":"2024-09-14T03:34:03.508776+0000","last_undegraded":"2024-09-14T03:34:03.508776+0000","last_fullsized":"2024-09-14T03:34:03.508776+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:35.627973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.509308+0000","last_change":"2024-09-14T03:33:59.395532+0000","last_active":"2024-09-14T03:34:04.509308+0000","last_peered":"2024-09-14T03:34:04.509308+0000","last_clean":"2024-09-14T03:34:04.509308+0000","last_became_active":"2024-09-14T03:33:59.395255+0000","last_became_peered":"2024-09-14T03:33:59.395255+0000","last_unstale":"2024-09-14T03:34:04.509308+0000","last_undegraded":"2024-09-14T03:34:04.509308+0000","last_fullsized":"2024-09-14T03:34:04.509308+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:51.477913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479309+0000","last_change":"2024-09-14T03:34:02.520744+0000","last_active":"2024-09-14T03:34:04.479309+0000","last_peered":"2024-09-14T03:34:04.479309+0000","last_clean":"2024-09-14T03:34:04.479309+0000","last_became_active":"2024-09-14T03:34:02.520420+0000","last_became_peered":"2024-09-14T03:34:02.520420+0000","last_unstale":"2024-09-14T03:34:04.479309+0000","last_undegraded":"2024-09-14T03:34:04.479309+0000","last_fullsized":"2024-09-14T03:34:04.479309+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:32:43.573943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507667+0000","last_change":"2024-09-14T03:33:59.399299+0000","last_active":"2024-09-14T03:34:03.507667+0000","last_peered":"2024-09-14T03:34:03.507667+0000","last_clean":"2024-09-14T03:34:03.507667+0000","last_became_active":"2024-09-14T03:33:59.399008+0000","last_became_peered":"2024-09-14T03:33:59.399008+0000","last_unstale":"2024-09-14T03:34:03.507667+0000","last_undegraded":"2024-09-14T03:34:03.507667+0000","last_fullsized":"2024-09-14T03:34:03.507667+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:16:14.935721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508396+0000","last_change":"2024-09-14T03:34:02.521443+0000","last_active":"2024-09-14T03:34:03.508396+0000","last_peered":"2024-09-14T03:34:03.508396+0000","last_clean":"2024-09-14T03:34:03.508396+0000","last_became_active":"2024-09-14T03:34:02.519164+0000","last_became_peered":"2024-09-14T03:34:02.519164+0000","last_unstale":"2024-09-14T03:34:03.508396+0000","last_undegraded":"2024-09-14T03:34:03.508396+0000","last_fullsized":"2024-09-14T03:34:03.508396+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:28.199628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":23,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508242+0000","last_change":"2024-09-14T03:33:36.972974+0000","last_active":"2024-09-14T03:34:03.508242+0000","last_peered":"2024-09-14T03:34:03.508242+0000","last_clean":"2024-09-14T03:34:03.508242+0000","last_became_active":"2024-09-14T03:33:36.967695+0000","last_became_peered":"2024-09-14T03:33:36.967695+0000","last_unstale":"2024-09-14T03:34:03.508242+0000","last_undegraded":"2024-09-14T03:34:03.508242+0000","last_fullsized":"2024-09-14T03:34:03.508242+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_clean_scrub_stamp":"2024-09-14T03:31:39.719769+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:28:24.258573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478853+0000","last_change":"2024-09-14T03:33:59.406222+0000","last_active":"2024-09-14T03:34:04.478853+0000","last_peered":"2024-09-14T03:34:04.478853+0000","last_clean":"2024-09-14T03:34:04.478853+0000","last_became_active":"2024-09-14T03:33:59.405473+0000","last_became_peered":"2024-09-14T03:33:59.405473+0000","last_unstale":"2024-09-14T03:34:04.478853+0000","last_undegraded":"2024-09-14T03:34:04.478853+0000","last_fullsized":"2024-09-14T03:34:04.478853+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:26.326984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479087+0000","last_change":"2024-09-14T03:34:02.512416+0000","last_active":"2024-09-14T03:34:04.479087+0000","last_peered":"2024-09-14T03:34:04.479087+0000","last_clean":"2024-09-14T03:34:04.479087+0000","last_became_active":"2024-09-14T03:34:02.512135+0000","last_became_peered":"2024-09-14T03:34:02.512135+0000","last_unstale":"2024-09-14T03:34:04.479087+0000","last_undegraded":"2024-09-14T03:34:04.479087+0000","last_fullsized":"2024-09-14T03:34:04.479087+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:49.847882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508093+0000","last_change":"2024-09-14T03:33:59.409150+0000","last_active":"2024-09-14T03:34:03.508093+0000","last_peered":"2024-09-14T03:34:03.508093+0000","last_clean":"2024-09-14T03:34:03.508093+0000","last_became_active":"2024-09-14T03:33:59.408622+0000","last_became_peered":"2024-09-14T03:33:59.408622+0000","last_unstale":"2024-09-14T03:34:03.508093+0000","last_undegraded":"2024-09-14T03:34:03.508093+0000","last_fullsized":"2024-09-14T03:34:03.508093+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:11:03.508216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508080+0000","last_change":"2024-09-14T03:34:02.517068+0000","last_active":"2024-09-14T03:34:03.508080+0000","last_peered":"2024-09-14T03:34:03.508080+0000","last_clean":"2024-09-14T03:34:03.508080+0000","last_became_active":"2024-09-14T03:34:02.510158+0000","last_became_peered":"2024-09-14T03:34:02.510158+0000","last_unstale":"2024-09-14T03:34:03.508080+0000","last_undegraded":"2024-09-14T03:34:03.508080+0000","last_fullsized":"2024-09-14T03:34:03.508080+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:22.709122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507560+0000","last_change":"2024-09-14T03:33:59.399987+0000","last_active":"2024-09-14T03:34:03.507560+0000","last_peered":"2024-09-14T03:34:03.507560+0000","last_clean":"2024-09-14T03:34:03.507560+0000","last_became_active":"2024-09-14T03:33:59.399698+0000","last_became_peered":"2024-09-14T03:33:59.399698+0000","last_unstale":"2024-09-14T03:34:03.507560+0000","last_undegraded":"2024-09-14T03:34:03.507560+0000","last_fullsized":"2024-09-14T03:34:03.507560+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:58.806927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478614+0000","last_change":"2024-09-14T03:34:02.515266+0000","last_active":"2024-09-14T03:34:04.478614+0000","last_peered":"2024-09-14T03:34:04.478614+0000","last_clean":"2024-09-14T03:34:04.478614+0000","last_became_active":"2024-09-14T03:34:02.515012+0000","last_became_peered":"2024-09-14T03:34:02.515012+0000","last_unstale":"2024-09-14T03:34:04.478614+0000","last_undegraded":"2024-09-14T03:34:04.478614+0000","last_fullsized":"2024-09-14T03:34:04.478614+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:52:23.510282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479727+0000","last_change":"2024-09-14T03:33:59.399736+0000","last_active":"2024-09-14T03:34:04.479727+0000","last_peered":"2024-09-14T03:34:04.479727+0000","last_clean":"2024-09-14T03:34:04.479727+0000","last_became_active":"2024-09-14T03:33:59.399434+0000","last_became_peered":"2024-09-14T03:33:59.399434+0000","last_unstale":"2024-09-14T03:34:04.479727+0000","last_undegraded":"2024-09-14T03:34:04.479727+0000","last_fullsized":"2024-09-14T03:34:04.479727+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:12.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479707+0000","last_change":"2024-09-14T03:34:02.507742+0000","last_active":"2024-09-14T03:34:04.479707+0000","last_peered":"2024-09-14T03:34:04.479707+0000","last_clean":"2024-09-14T03:34:04.479707+0000","last_became_active":"2024-09-14T03:34:02.507452+0000","last_became_peered":"2024-09-14T03:34:02.507452+0000","last_unstale":"2024-09-14T03:34:04.479707+0000","last_undegraded":"2024-09-14T03:34:04.479707+0000","last_fullsized":"2024-09-14T03:34:04.479707+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:50.407482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478910+0000","last_change":"2024-09-14T03:33:59.400498+0000","last_active":"2024-09-14T03:34:04.478910+0000","last_peered":"2024-09-14T03:34:04.478910+0000","last_clean":"2024-09-14T03:34:04.478910+0000","last_became_active":"2024-09-14T03:33:59.400204+0000","last_became_peered":"2024-09-14T03:33:59.400204+0000","last_unstale":"2024-09-14T03:34:04.478910+0000","last_undegraded":"2024-09-14T03:34:04.478910+0000","last_fullsized":"2024-09-14T03:34:04.478910+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:07:48.735887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479622+0000","last_change":"2024-09-14T03:34:02.513933+0000","last_active":"2024-09-14T03:34:04.479622+0000","last_peered":"2024-09-14T03:34:04.479622+0000","last_clean":"2024-09-14T03:34:04.479622+0000","last_became_active":"2024-09-14T03:34:02.513689+0000","last_became_peered":"2024-09-14T03:34:02.513689+0000","last_unstale":"2024-09-14T03:34:04.479622+0000","last_undegraded":"2024-09-14T03:34:04.479622+0000","last_fullsized":"2024-09-14T03:34:04.479622+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:23.791135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508518+0000","last_change":"2024-09-14T03:33:59.406992+0000","last_active":"2024-09-14T03:34:03.508518+0000","last_peered":"2024-09-14T03:34:03.508518+0000","last_clean":"2024-09-14T03:34:03.508518+0000","last_became_active":"2024-09-14T03:33:59.406729+0000","last_became_peered":"2024-09-14T03:33:59.406729+0000","last_unstale":"2024-09-14T03:34:03.508518+0000","last_undegraded":"2024-09-14T03:34:03.508518+0000","last_fullsized":"2024-09-14T03:34:03.508518+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:20:58.830368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478822+0000","last_change":"2024-09-14T03:34:02.513000+0000","last_active":"2024-09-14T03:34:04.478822+0000","last_peered":"2024-09-14T03:34:04.478822+0000","last_clean":"2024-09-14T03:34:04.478822+0000","last_became_active":"2024-09-14T03:34:02.512679+0000","last_became_peered":"2024-09-14T03:34:02.512679+0000","last_unstale":"2024-09-14T03:34:04.478822+0000","last_undegraded":"2024-09-14T03:34:04.478822+0000","last_fullsized":"2024-09-14T03:34:04.478822+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:43:48.821069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.510278+0000","last_change":"2024-09-14T03:33:59.397629+0000","last_active":"2024-09-14T03:34:04.510278+0000","last_peered":"2024-09-14T03:34:04.510278+0000","last_clean":"2024-09-14T03:34:04.510278+0000","last_became_active":"2024-09-14T03:33:59.397377+0000","last_became_peered":"2024-09-14T03:33:59.397377+0000","last_unstale":"2024-09-14T03:34:04.510278+0000","last_undegraded":"2024-09-14T03:34:04.510278+0000","last_fullsized":"2024-09-14T03:34:04.510278+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:42:02.914679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479450+0000","last_change":"2024-09-14T03:34:02.517598+0000","last_active":"2024-09-14T03:34:04.479450+0000","last_peered":"2024-09-14T03:34:04.479450+0000","last_clean":"2024-09-14T03:34:04.479450+0000","last_became_active":"2024-09-14T03:34:02.517367+0000","last_became_peered":"2024-09-14T03:34:02.517367+0000","last_unstale":"2024-09-14T03:34:04.479450+0000","last_undegraded":"2024-09-14T03:34:04.479450+0000","last_fullsized":"2024-09-14T03:34:04.479450+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:44.727080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.520324+0000","last_change":"2024-09-14T03:33:59.405301+0000","last_active":"2024-09-14T03:34:04.520324+0000","last_peered":"2024-09-14T03:34:04.520324+0000","last_clean":"2024-09-14T03:34:04.520324+0000","last_became_active":"2024-09-14T03:33:59.404994+0000","last_became_peered":"2024-09-14T03:33:59.404994+0000","last_unstale":"2024-09-14T03:34:04.520324+0000","last_undegraded":"2024-09-14T03:34:04.520324+0000","last_fullsized":"2024-09-14T03:34:04.520324+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:52:19.415253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508018+0000","last_change":"2024-09-14T03:34:02.516570+0000","last_active":"2024-09-14T03:34:03.508018+0000","last_peered":"2024-09-14T03:34:03.508018+0000","last_clean":"2024-09-14T03:34:03.508018+0000","last_became_active":"2024-09-14T03:34:02.516212+0000","last_became_peered":"2024-09-14T03:34:02.516212+0000","last_unstale":"2024-09-14T03:34:03.508018+0000","last_undegraded":"2024-09-14T03:34:03.508018+0000","last_fullsized":"2024-09-14T03:34:03.508018+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:33:23.282986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508340+0000","last_change":"2024-09-14T03:33:59.408805+0000","last_active":"2024-09-14T03:34:03.508340+0000","last_peered":"2024-09-14T03:34:03.508340+0000","last_clean":"2024-09-14T03:34:03.508340+0000","last_became_active":"2024-09-14T03:33:59.408521+0000","last_became_peered":"2024-09-14T03:33:59.408521+0000","last_unstale":"2024-09-14T03:34:03.508340+0000","last_undegraded":"2024-09-14T03:34:03.508340+0000","last_fullsized":"2024-09-14T03:34:03.508340+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:20:00.847563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478731+0000","last_change":"2024-09-14T03:34:02.511411+0000","last_active":"2024-09-14T03:34:04.478731+0000","last_peered":"2024-09-14T03:34:04.478731+0000","last_clean":"2024-09-14T03:34:04.478731+0000","last_became_active":"2024-09-14T03:34:02.511131+0000","last_became_peered":"2024-09-14T03:34:02.511131+0000","last_unstale":"2024-09-14T03:34:04.478731+0000","last_undegraded":"2024-09-14T03:34:04.478731+0000","last_fullsized":"2024-09-14T03:34:04.478731+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:23.033860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479335+0000","last_change":"2024-09-14T03:33:59.400483+0000","last_active":"2024-09-14T03:34:04.479335+0000","last_peered":"2024-09-14T03:34:04.479335+0000","last_clean":"2024-09-14T03:34:04.479335+0000","last_became_active":"2024-09-14T03:33:59.400172+0000","last_became_peered":"2024-09-14T03:33:59.400172+0000","last_unstale":"2024-09-14T03:34:04.479335+0000","last_undegraded":"2024-09-14T03:34:04.479335+0000","last_fullsized":"2024-09-14T03:34:04.479335+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:40.756478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478905+0000","last_change":"2024-09-14T03:34:02.505712+0000","last_active":"2024-09-14T03:34:04.478905+0000","last_peered":"2024-09-14T03:34:04.478905+0000","last_clean":"2024-09-14T03:34:04.478905+0000","last_became_active":"2024-09-14T03:34:02.505460+0000","last_became_peered":"2024-09-14T03:34:02.505460+0000","last_unstale":"2024-09-14T03:34:04.478905+0000","last_undegraded":"2024-09-14T03:34:04.478905+0000","last_fullsized":"2024-09-14T03:34:04.478905+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:20.752579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479364+0000","last_change":"2024-09-14T03:33:59.405640+0000","last_active":"2024-09-14T03:34:04.479364+0000","last_peered":"2024-09-14T03:34:04.479364+0000","last_clean":"2024-09-14T03:34:04.479364+0000","last_became_active":"2024-09-14T03:33:59.405382+0000","last_became_peered":"2024-09-14T03:33:59.405382+0000","last_unstale":"2024-09-14T03:34:04.479364+0000","last_undegraded":"2024-09-14T03:34:04.479364+0000","last_fullsized":"2024-09-14T03:34:04.479364+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:36.350128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479036+0000","last_change":"2024-09-14T03:34:02.516581+0000","last_active":"2024-09-14T03:34:04.479036+0000","last_peered":"2024-09-14T03:34:04.479036+0000","last_clean":"2024-09-14T03:34:04.479036+0000","last_became_active":"2024-09-14T03:34:02.516335+0000","last_became_peered":"2024-09-14T03:34:02.516335+0000","last_unstale":"2024-09-14T03:34:04.479036+0000","last_undegraded":"2024-09-14T03:34:04.479036+0000","last_fullsized":"2024-09-14T03:34:04.479036+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:06.070599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.521521+0000","last_change":"2024-09-14T03:33:59.408573+0000","last_active":"2024-09-14T03:34:04.521521+0000","last_peered":"2024-09-14T03:34:04.521521+0000","last_clean":"2024-09-14T03:34:04.521521+0000","last_became_active":"2024-09-14T03:33:59.408264+0000","last_became_peered":"2024-09-14T03:33:59.408264+0000","last_unstale":"2024-09-14T03:34:04.521521+0000","last_undegraded":"2024-09-14T03:34:04.521521+0000","last_fullsized":"2024-09-14T03:34:04.521521+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:30.937449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547274+0000","last_change":"2024-09-14T03:34:02.516408+0000","last_active":"2024-09-14T03:34:03.547274+0000","last_peered":"2024-09-14T03:34:03.547274+0000","last_clean":"2024-09-14T03:34:03.547274+0000","last_became_active":"2024-09-14T03:34:02.516119+0000","last_became_peered":"2024-09-14T03:34:02.516119+0000","last_unstale":"2024-09-14T03:34:03.547274+0000","last_undegraded":"2024-09-14T03:34:03.547274+0000","last_fullsized":"2024-09-14T03:34:03.547274+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:38:16.570920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479461+0000","last_change":"2024-09-14T03:34:02.512557+0000","last_active":"2024-09-14T03:34:04.479461+0000","last_peered":"2024-09-14T03:34:04.479461+0000","last_clean":"2024-09-14T03:34:04.479461+0000","last_became_active":"2024-09-14T03:34:02.512183+0000","last_became_peered":"2024-09-14T03:34:02.512183+0000","last_unstale":"2024-09-14T03:34:04.479461+0000","last_undegraded":"2024-09-14T03:34:04.479461+0000","last_fullsized":"2024-09-14T03:34:04.479461+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:05:14.073579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.522438+0000","last_change":"2024-09-14T03:33:59.406333+0000","last_active":"2024-09-14T03:34:04.522438+0000","last_peered":"2024-09-14T03:34:04.522438+0000","last_clean":"2024-09-14T03:34:04.522438+0000","last_became_active":"2024-09-14T03:33:59.406065+0000","last_became_peered":"2024-09-14T03:33:59.406065+0000","last_unstale":"2024-09-14T03:34:04.522438+0000","last_undegraded":"2024-09-14T03:34:04.522438+0000","last_fullsized":"2024-09-14T03:34:04.522438+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:04.009889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479258+0000","last_change":"2024-09-14T03:34:02.508363+0000","last_active":"2024-09-14T03:34:04.479258+0000","last_peered":"2024-09-14T03:34:04.479258+0000","last_clean":"2024-09-14T03:34:04.479258+0000","last_became_active":"2024-09-14T03:34:02.508086+0000","last_became_peered":"2024-09-14T03:34:02.508086+0000","last_unstale":"2024-09-14T03:34:04.479258+0000","last_undegraded":"2024-09-14T03:34:04.479258+0000","last_fullsized":"2024-09-14T03:34:04.479258+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:47.194036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.519684+0000","last_change":"2024-09-14T03:33:59.406945+0000","last_active":"2024-09-14T03:34:04.519684+0000","last_peered":"2024-09-14T03:34:04.519684+0000","last_clean":"2024-09-14T03:34:04.519684+0000","last_became_active":"2024-09-14T03:33:59.406662+0000","last_became_peered":"2024-09-14T03:33:59.406662+0000","last_unstale":"2024-09-14T03:34:04.519684+0000","last_undegraded":"2024-09-14T03:34:04.519684+0000","last_fullsized":"2024-09-14T03:34:04.519684+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:46:42.749128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508697+0000","last_change":"2024-09-14T03:34:02.510922+0000","last_active":"2024-09-14T03:34:03.508697+0000","last_peered":"2024-09-14T03:34:03.508697+0000","last_clean":"2024-09-14T03:34:03.508697+0000","last_became_active":"2024-09-14T03:34:02.510663+0000","last_became_peered":"2024-09-14T03:34:02.510663+0000","last_unstale":"2024-09-14T03:34:03.508697+0000","last_undegraded":"2024-09-14T03:34:03.508697+0000","last_fullsized":"2024-09-14T03:34:03.508697+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:00:02.779787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.517651+0000","last_change":"2024-09-14T03:33:59.398992+0000","last_active":"2024-09-14T03:34:04.517651+0000","last_peered":"2024-09-14T03:34:04.517651+0000","last_clean":"2024-09-14T03:34:04.517651+0000","last_became_active":"2024-09-14T03:33:59.398725+0000","last_became_peered":"2024-09-14T03:33:59.398725+0000","last_unstale":"2024-09-14T03:34:04.517651+0000","last_undegraded":"2024-09-14T03:34:04.517651+0000","last_fullsized":"2024-09-14T03:34:04.517651+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:50:20.745864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478974+0000","last_change":"2024-09-14T03:34:02.514810+0000","last_active":"2024-09-14T03:34:04.478974+0000","last_peered":"2024-09-14T03:34:04.478974+0000","last_clean":"2024-09-14T03:34:04.478974+0000","last_became_active":"2024-09-14T03:34:02.514500+0000","last_became_peered":"2024-09-14T03:34:02.514500+0000","last_unstale":"2024-09-14T03:34:04.478974+0000","last_undegraded":"2024-09-14T03:34:04.478974+0000","last_fullsized":"2024-09-14T03:34:04.478974+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:20:21.110046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.521464+0000","last_change":"2024-09-14T03:33:59.397898+0000","last_active":"2024-09-14T03:34:04.521464+0000","last_peered":"2024-09-14T03:34:04.521464+0000","last_clean":"2024-09-14T03:34:04.521464+0000","last_became_active":"2024-09-14T03:33:59.396670+0000","last_became_peered":"2024-09-14T03:33:59.396670+0000","last_unstale":"2024-09-14T03:34:04.521464+0000","last_undegraded":"2024-09-14T03:34:04.521464+0000","last_fullsized":"2024-09-14T03:34:04.521464+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:46.652243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397515,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1175552,"data_stored":831572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42799999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63900000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]}]},{"osd":6,"up_from":43,"seq":184683593744,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298512,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446640,"statfs":{"total":95995035648,"available":95689359360,"internally_reserved":0,"allocated":1175552,"data_stored":832217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52300000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.442},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.33000000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.38800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.373}]}]},{"osd":5,"up_from":37,"seq":158913789973,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":581632,"data_stored":241695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.64400000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.497},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.436}]},{"osd":2,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.52900000000000003}]},{"osd":3,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.69199999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67100000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986201,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":589824,"data_stored":241308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.46899999999999997}]},{"osd":1,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.627}]},{"osd":2,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.57699999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.66400000000000003}]},{"osd":5,"last update":"Sat Sep 14 03:33:47 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.68400000000000005}]},{"osd":6,"last update":"Sat Sep 14 03:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.75600000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73299999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149726,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":593920,"data_stored":242268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:34:08 2024","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.35399999999999998,"15min":0.35799999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.439,"5min":0.502,"15min":0.502},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.41899999999999998,"15min":0.42699999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.48299999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.311}]},{"osd":1,"last update":"Sat Sep 14 03:34:08 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.38600000000000001,"15min":0.39000000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.501,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.38900000000000001,"15min":0.39500000000000002},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.52200000000000002,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:34:08 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.48099999999999998,"5min":0.54600000000000004,"15min":0.54600000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.39800000000000002,"15min":0.40400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.46100000000000002,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.26400000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:33:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.57899999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.55900000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61299999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411363,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":560,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":573440,"data_stored":240612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:33:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.40999999999999998,"15min":0.40799999999999997},"min":{"1min":0.219,"5min":0.16300000000000001,"15min":0.16300000000000001},"max":{"1min":0.59599999999999997,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.435,"15min":0.432},"min":{"1min":0.27600000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.68999999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 14 03:33:43 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.38500000000000001,"15min":0.38100000000000001},"min":{"1min":0.28100000000000003,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.39700000000000002},"min":{"1min":0.29299999999999998,"5min":0.224,"15min":0.224},"max":{"1min":0.58399999999999996,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:34:03 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.443,"15min":0.442},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.28199999999999997}]},{"osd":4,"last update":"Sat Sep 14 03:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.374}]},{"osd":5,"last update":"Sat Sep 14 03:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 14 03:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.36099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39400000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574887,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":556,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":569344,"data_stored":239827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:33:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.32700000000000001,"15min":0.315},"min":{"1min":0.29499999999999998,"5min":0.214,"15min":0.214},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.307},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.33900000000000002,"15min":0.33100000000000002},"min":{"1min":0.312,"5min":0.214,"15min":0.214},"max":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39800000000000002,"15min":0.39900000000000002},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.52000000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.41899999999999998,"15min":0.41499999999999998},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 14 03:34:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.45800000000000002,"15min":0.46600000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.27400000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.438}]},{"osd":5,"last update":"Sat Sep 14 03:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.374},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.496}]},{"osd":6,"last update":"Sat Sep 14 03:34:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.752,"5min":0.752,"15min":0.752},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":4.9050000000000002,"5min":4.9050000000000002,"15min":4.9050000000000002},"last":0.42299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1156,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1183744,"data_stored":831696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.32000000000000001,"15min":0.311},"min":{"1min":0.224,"5min":0.192,"15min":0.192},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.33600000000000002,"15min":0.32600000000000001},"min":{"1min":0.29199999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.35599999999999998},"min":{"1min":0.28100000000000003,"5min":0.253,"15min":0.253},"max":{"1min":0.53400000000000003,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.379,"15min":0.373},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.318}]},{"osd":3,"last update":"Sat Sep 14 03:34:07 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41799999999999998,"15min":0.41899999999999998},"min":{"1min":0.29299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.439},{"interface":"front","average":{"1min":0.435,"5min":0.45400000000000001,"15min":0.45700000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.379}]},{"osd":4,"last update":"Sat Sep 14 03:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.58299999999999996}]},{"osd":5,"last update":"Sat Sep 14 03:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.624},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.74399999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:34:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.5449999999999999,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.60999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:34:24.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:34:24.753 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:24 smithi110 ceph-mon[35374]: from='client.24386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:34:25.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:24 smithi110 ceph-mon[27365]: from='client.24386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:34:25.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:24 smithi195 ceph-mon[30695]: from='client.24386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:34:26.137 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:25 smithi110 ceph-mon[35374]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:26.137 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:25 smithi110 ceph-mon[27365]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:25 smithi195 ceph-mon[30695]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:26.455 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:34:26.456 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2024-09-14T03:34:27.107 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":194,"stamp":"2024-09-14T03:34:24.833011+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385188,"kb_used_data":6292,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576028,"statfs":{"total":767960285184,"available":765517852672,"internally_reserved":0,"allocated":6443008,"data_stored":3701195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003104"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508680+0000","last_change":"2024-09-14T03:34:02.519367+0000","last_active":"2024-09-14T03:34:03.508680+0000","last_peered":"2024-09-14T03:34:03.508680+0000","last_clean":"2024-09-14T03:34:03.508680+0000","last_became_active":"2024-09-14T03:34:02.518530+0000","last_became_peered":"2024-09-14T03:34:02.518530+0000","last_unstale":"2024-09-14T03:34:03.508680+0000","last_undegraded":"2024-09-14T03:34:03.508680+0000","last_fullsized":"2024-09-14T03:34:03.508680+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:02:36.205002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479156+0000","last_change":"2024-09-14T03:34:02.515212+0000","last_active":"2024-09-14T03:34:04.479156+0000","last_peered":"2024-09-14T03:34:04.479156+0000","last_clean":"2024-09-14T03:34:04.479156+0000","last_became_active":"2024-09-14T03:34:02.514951+0000","last_became_peered":"2024-09-14T03:34:02.514951+0000","last_unstale":"2024-09-14T03:34:04.479156+0000","last_undegraded":"2024-09-14T03:34:04.479156+0000","last_fullsized":"2024-09-14T03:34:04.479156+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:30.742131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508290+0000","last_change":"2024-09-14T03:34:02.504777+0000","last_active":"2024-09-14T03:34:03.508290+0000","last_peered":"2024-09-14T03:34:03.508290+0000","last_clean":"2024-09-14T03:34:03.508290+0000","last_became_active":"2024-09-14T03:34:02.504550+0000","last_became_peered":"2024-09-14T03:34:02.504550+0000","last_unstale":"2024-09-14T03:34:03.508290+0000","last_undegraded":"2024-09-14T03:34:03.508290+0000","last_fullsized":"2024-09-14T03:34:03.508290+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:59.864492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479137+0000","last_change":"2024-09-14T03:34:02.521444+0000","last_active":"2024-09-14T03:34:04.479137+0000","last_peered":"2024-09-14T03:34:04.479137+0000","last_clean":"2024-09-14T03:34:04.479137+0000","last_became_active":"2024-09-14T03:34:02.521149+0000","last_became_peered":"2024-09-14T03:34:02.521149+0000","last_unstale":"2024-09-14T03:34:04.479137+0000","last_undegraded":"2024-09-14T03:34:04.479137+0000","last_fullsized":"2024-09-14T03:34:04.479137+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:10.004625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508126+0000","last_change":"2024-09-14T03:34:02.513133+0000","last_active":"2024-09-14T03:34:03.508126+0000","last_peered":"2024-09-14T03:34:03.508126+0000","last_clean":"2024-09-14T03:34:03.508126+0000","last_became_active":"2024-09-14T03:34:02.512580+0000","last_became_peered":"2024-09-14T03:34:02.512580+0000","last_unstale":"2024-09-14T03:34:03.508126+0000","last_undegraded":"2024-09-14T03:34:03.508126+0000","last_fullsized":"2024-09-14T03:34:03.508126+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:03.077106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479529+0000","last_change":"2024-09-14T03:34:02.508230+0000","last_active":"2024-09-14T03:34:04.479529+0000","last_peered":"2024-09-14T03:34:04.479529+0000","last_clean":"2024-09-14T03:34:04.479529+0000","last_became_active":"2024-09-14T03:34:02.507921+0000","last_became_peered":"2024-09-14T03:34:02.507921+0000","last_unstale":"2024-09-14T03:34:04.479529+0000","last_undegraded":"2024-09-14T03:34:04.479529+0000","last_fullsized":"2024-09-14T03:34:04.479529+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:37.418870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507913+0000","last_change":"2024-09-14T03:34:02.515319+0000","last_active":"2024-09-14T03:34:03.507913+0000","last_peered":"2024-09-14T03:34:03.507913+0000","last_clean":"2024-09-14T03:34:03.507913+0000","last_became_active":"2024-09-14T03:34:02.515022+0000","last_became_peered":"2024-09-14T03:34:02.515022+0000","last_unstale":"2024-09-14T03:34:03.507913+0000","last_undegraded":"2024-09-14T03:34:03.507913+0000","last_fullsized":"2024-09-14T03:34:03.507913+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:51:51.089278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478715+0000","last_change":"2024-09-14T03:34:02.522349+0000","last_active":"2024-09-14T03:34:04.478715+0000","last_peered":"2024-09-14T03:34:04.478715+0000","last_clean":"2024-09-14T03:34:04.478715+0000","last_became_active":"2024-09-14T03:34:02.522063+0000","last_became_peered":"2024-09-14T03:34:02.522063+0000","last_unstale":"2024-09-14T03:34:04.478715+0000","last_undegraded":"2024-09-14T03:34:04.478715+0000","last_fullsized":"2024-09-14T03:34:04.478715+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:15.378220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508868+0000","last_change":"2024-09-14T03:34:02.512031+0000","last_active":"2024-09-14T03:34:03.508868+0000","last_peered":"2024-09-14T03:34:03.508868+0000","last_clean":"2024-09-14T03:34:03.508868+0000","last_became_active":"2024-09-14T03:34:02.511364+0000","last_became_peered":"2024-09-14T03:34:02.511364+0000","last_unstale":"2024-09-14T03:34:03.508868+0000","last_undegraded":"2024-09-14T03:34:03.508868+0000","last_fullsized":"2024-09-14T03:34:03.508868+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:54:15.711934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547595+0000","last_change":"2024-09-14T03:34:02.518932+0000","last_active":"2024-09-14T03:34:03.547595+0000","last_peered":"2024-09-14T03:34:03.547595+0000","last_clean":"2024-09-14T03:34:03.547595+0000","last_became_active":"2024-09-14T03:34:02.518575+0000","last_became_peered":"2024-09-14T03:34:02.518575+0000","last_unstale":"2024-09-14T03:34:03.547595+0000","last_undegraded":"2024-09-14T03:34:03.547595+0000","last_fullsized":"2024-09-14T03:34:03.547595+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:01:11.929183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547443+0000","last_change":"2024-09-14T03:34:02.517977+0000","last_active":"2024-09-14T03:34:03.547443+0000","last_peered":"2024-09-14T03:34:03.547443+0000","last_clean":"2024-09-14T03:34:03.547443+0000","last_became_active":"2024-09-14T03:34:02.517385+0000","last_became_peered":"2024-09-14T03:34:02.517385+0000","last_unstale":"2024-09-14T03:34:03.547443+0000","last_undegraded":"2024-09-14T03:34:03.547443+0000","last_fullsized":"2024-09-14T03:34:03.547443+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:50.900533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479601+0000","last_change":"2024-09-14T03:34:02.510949+0000","last_active":"2024-09-14T03:34:04.479601+0000","last_peered":"2024-09-14T03:34:04.479601+0000","last_clean":"2024-09-14T03:34:04.479601+0000","last_became_active":"2024-09-14T03:34:02.510519+0000","last_became_peered":"2024-09-14T03:34:02.510519+0000","last_unstale":"2024-09-14T03:34:04.479601+0000","last_undegraded":"2024-09-14T03:34:04.479601+0000","last_fullsized":"2024-09-14T03:34:04.479601+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:06:16.860908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478947+0000","last_change":"2024-09-14T03:34:02.512463+0000","last_active":"2024-09-14T03:34:04.478947+0000","last_peered":"2024-09-14T03:34:04.478947+0000","last_clean":"2024-09-14T03:34:04.478947+0000","last_became_active":"2024-09-14T03:34:02.512201+0000","last_became_peered":"2024-09-14T03:34:02.512201+0000","last_unstale":"2024-09-14T03:34:04.478947+0000","last_undegraded":"2024-09-14T03:34:04.478947+0000","last_fullsized":"2024-09-14T03:34:04.478947+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:46.716482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508412+0000","last_change":"2024-09-14T03:34:02.513215+0000","last_active":"2024-09-14T03:34:03.508412+0000","last_peered":"2024-09-14T03:34:03.508412+0000","last_clean":"2024-09-14T03:34:03.508412+0000","last_became_active":"2024-09-14T03:34:02.512773+0000","last_became_peered":"2024-09-14T03:34:02.512773+0000","last_unstale":"2024-09-14T03:34:03.508412+0000","last_undegraded":"2024-09-14T03:34:03.508412+0000","last_fullsized":"2024-09-14T03:34:03.508412+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:27:04.454480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478731+0000","last_change":"2024-09-14T03:34:02.503785+0000","last_active":"2024-09-14T03:34:04.478731+0000","last_peered":"2024-09-14T03:34:04.478731+0000","last_clean":"2024-09-14T03:34:04.478731+0000","last_became_active":"2024-09-14T03:34:02.503212+0000","last_became_peered":"2024-09-14T03:34:02.503212+0000","last_unstale":"2024-09-14T03:34:04.478731+0000","last_undegraded":"2024-09-14T03:34:04.478731+0000","last_fullsized":"2024-09-14T03:34:04.478731+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:15.408242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478728+0000","last_change":"2024-09-14T03:34:02.510927+0000","last_active":"2024-09-14T03:34:04.478728+0000","last_peered":"2024-09-14T03:34:04.478728+0000","last_clean":"2024-09-14T03:34:04.478728+0000","last_became_active":"2024-09-14T03:34:02.510575+0000","last_became_peered":"2024-09-14T03:34:02.510575+0000","last_unstale":"2024-09-14T03:34:04.478728+0000","last_undegraded":"2024-09-14T03:34:04.478728+0000","last_fullsized":"2024-09-14T03:34:04.478728+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:14.173783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479228+0000","last_change":"2024-09-14T03:34:02.507186+0000","last_active":"2024-09-14T03:34:04.479228+0000","last_peered":"2024-09-14T03:34:04.479228+0000","last_clean":"2024-09-14T03:34:04.479228+0000","last_became_active":"2024-09-14T03:34:02.506907+0000","last_became_peered":"2024-09-14T03:34:02.506907+0000","last_unstale":"2024-09-14T03:34:04.479228+0000","last_undegraded":"2024-09-14T03:34:04.479228+0000","last_fullsized":"2024-09-14T03:34:04.479228+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:10:47.570717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508073+0000","last_change":"2024-09-14T03:34:02.519104+0000","last_active":"2024-09-14T03:34:03.508073+0000","last_peered":"2024-09-14T03:34:03.508073+0000","last_clean":"2024-09-14T03:34:03.508073+0000","last_became_active":"2024-09-14T03:34:02.518795+0000","last_became_peered":"2024-09-14T03:34:02.518795+0000","last_unstale":"2024-09-14T03:34:03.508073+0000","last_undegraded":"2024-09-14T03:34:03.508073+0000","last_fullsized":"2024-09-14T03:34:03.508073+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:34:49.356603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508498+0000","last_change":"2024-09-14T03:34:02.521848+0000","last_active":"2024-09-14T03:34:03.508498+0000","last_peered":"2024-09-14T03:34:03.508498+0000","last_clean":"2024-09-14T03:34:03.508498+0000","last_became_active":"2024-09-14T03:34:02.521559+0000","last_became_peered":"2024-09-14T03:34:02.521559+0000","last_unstale":"2024-09-14T03:34:03.508498+0000","last_undegraded":"2024-09-14T03:34:03.508498+0000","last_fullsized":"2024-09-14T03:34:03.508498+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:21:09.742073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547522+0000","last_change":"2024-09-14T03:34:02.515094+0000","last_active":"2024-09-14T03:34:03.547522+0000","last_peered":"2024-09-14T03:34:03.547522+0000","last_clean":"2024-09-14T03:34:03.547522+0000","last_became_active":"2024-09-14T03:34:02.514291+0000","last_became_peered":"2024-09-14T03:34:02.514291+0000","last_unstale":"2024-09-14T03:34:03.547522+0000","last_undegraded":"2024-09-14T03:34:03.547522+0000","last_fullsized":"2024-09-14T03:34:03.547522+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:39:52.842240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508310+0000","last_change":"2024-09-14T03:34:02.510758+0000","last_active":"2024-09-14T03:34:03.508310+0000","last_peered":"2024-09-14T03:34:03.508310+0000","last_clean":"2024-09-14T03:34:03.508310+0000","last_became_active":"2024-09-14T03:34:02.510485+0000","last_became_peered":"2024-09-14T03:34:02.510485+0000","last_unstale":"2024-09-14T03:34:03.508310+0000","last_undegraded":"2024-09-14T03:34:03.508310+0000","last_fullsized":"2024-09-14T03:34:03.508310+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:26:25.102759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479538+0000","last_change":"2024-09-14T03:34:02.511696+0000","last_active":"2024-09-14T03:34:04.479538+0000","last_peered":"2024-09-14T03:34:04.479538+0000","last_clean":"2024-09-14T03:34:04.479538+0000","last_became_active":"2024-09-14T03:34:02.511289+0000","last_became_peered":"2024-09-14T03:34:02.511289+0000","last_unstale":"2024-09-14T03:34:04.479538+0000","last_undegraded":"2024-09-14T03:34:04.479538+0000","last_fullsized":"2024-09-14T03:34:04.479538+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:02:27.657439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478791+0000","last_change":"2024-09-14T03:34:02.517564+0000","last_active":"2024-09-14T03:34:04.478791+0000","last_peered":"2024-09-14T03:34:04.478791+0000","last_clean":"2024-09-14T03:34:04.478791+0000","last_became_active":"2024-09-14T03:34:02.517314+0000","last_became_peered":"2024-09-14T03:34:02.517314+0000","last_unstale":"2024-09-14T03:34:04.478791+0000","last_undegraded":"2024-09-14T03:34:04.478791+0000","last_fullsized":"2024-09-14T03:34:04.478791+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:08.360046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478817+0000","last_change":"2024-09-14T03:34:02.513865+0000","last_active":"2024-09-14T03:34:04.478817+0000","last_peered":"2024-09-14T03:34:04.478817+0000","last_clean":"2024-09-14T03:34:04.478817+0000","last_became_active":"2024-09-14T03:34:02.513008+0000","last_became_peered":"2024-09-14T03:34:02.513008+0000","last_unstale":"2024-09-14T03:34:04.478817+0000","last_undegraded":"2024-09-14T03:34:04.478817+0000","last_fullsized":"2024-09-14T03:34:04.478817+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:18:37.581596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479390+0000","last_change":"2024-09-14T03:34:02.517481+0000","last_active":"2024-09-14T03:34:04.479390+0000","last_peered":"2024-09-14T03:34:04.479390+0000","last_clean":"2024-09-14T03:34:04.479390+0000","last_became_active":"2024-09-14T03:34:02.517252+0000","last_became_peered":"2024-09-14T03:34:02.517252+0000","last_unstale":"2024-09-14T03:34:04.479390+0000","last_undegraded":"2024-09-14T03:34:04.479390+0000","last_fullsized":"2024-09-14T03:34:04.479390+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:29:15.209238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547669+0000","last_change":"2024-09-14T03:34:02.521201+0000","last_active":"2024-09-14T03:34:03.547669+0000","last_peered":"2024-09-14T03:34:03.547669+0000","last_clean":"2024-09-14T03:34:03.547669+0000","last_became_active":"2024-09-14T03:34:02.520764+0000","last_became_peered":"2024-09-14T03:34:02.520764+0000","last_unstale":"2024-09-14T03:34:03.547669+0000","last_undegraded":"2024-09-14T03:34:03.547669+0000","last_fullsized":"2024-09-14T03:34:03.547669+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:39:40.688146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479030+0000","last_change":"2024-09-14T03:34:02.504156+0000","last_active":"2024-09-14T03:34:04.479030+0000","last_peered":"2024-09-14T03:34:04.479030+0000","last_clean":"2024-09-14T03:34:04.479030+0000","last_became_active":"2024-09-14T03:34:02.503906+0000","last_became_peered":"2024-09-14T03:34:02.503906+0000","last_unstale":"2024-09-14T03:34:04.479030+0000","last_undegraded":"2024-09-14T03:34:04.479030+0000","last_fullsized":"2024-09-14T03:34:04.479030+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:33:57.350618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479315+0000","last_change":"2024-09-14T03:34:02.518555+0000","last_active":"2024-09-14T03:34:04.479315+0000","last_peered":"2024-09-14T03:34:04.479315+0000","last_clean":"2024-09-14T03:34:04.479315+0000","last_became_active":"2024-09-14T03:34:02.517920+0000","last_became_peered":"2024-09-14T03:34:02.517920+0000","last_unstale":"2024-09-14T03:34:04.479315+0000","last_undegraded":"2024-09-14T03:34:04.479315+0000","last_fullsized":"2024-09-14T03:34:04.479315+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:54.031933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508385+0000","last_change":"2024-09-14T03:34:02.515181+0000","last_active":"2024-09-14T03:34:03.508385+0000","last_peered":"2024-09-14T03:34:03.508385+0000","last_clean":"2024-09-14T03:34:03.508385+0000","last_became_active":"2024-09-14T03:34:02.514649+0000","last_became_peered":"2024-09-14T03:34:02.514649+0000","last_unstale":"2024-09-14T03:34:03.508385+0000","last_undegraded":"2024-09-14T03:34:03.508385+0000","last_fullsized":"2024-09-14T03:34:03.508385+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:05.770205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479663+0000","last_change":"2024-09-14T03:34:02.513294+0000","last_active":"2024-09-14T03:34:04.479663+0000","last_peered":"2024-09-14T03:34:04.479663+0000","last_clean":"2024-09-14T03:34:04.479663+0000","last_became_active":"2024-09-14T03:34:02.512998+0000","last_became_peered":"2024-09-14T03:34:02.512998+0000","last_unstale":"2024-09-14T03:34:04.479663+0000","last_undegraded":"2024-09-14T03:34:04.479663+0000","last_fullsized":"2024-09-14T03:34:04.479663+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:54.262830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507418+0000","last_change":"2024-09-14T03:34:02.512389+0000","last_active":"2024-09-14T03:34:03.507418+0000","last_peered":"2024-09-14T03:34:03.507418+0000","last_clean":"2024-09-14T03:34:03.507418+0000","last_became_active":"2024-09-14T03:34:02.511955+0000","last_became_peered":"2024-09-14T03:34:02.511955+0000","last_unstale":"2024-09-14T03:34:03.507418+0000","last_undegraded":"2024-09-14T03:34:03.507418+0000","last_fullsized":"2024-09-14T03:34:03.507418+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:30:31.454464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547061+0000","last_change":"2024-09-14T03:34:02.520047+0000","last_active":"2024-09-14T03:34:03.547061+0000","last_peered":"2024-09-14T03:34:03.547061+0000","last_clean":"2024-09-14T03:34:03.547061+0000","last_became_active":"2024-09-14T03:34:02.519764+0000","last_became_peered":"2024-09-14T03:34:02.519764+0000","last_unstale":"2024-09-14T03:34:03.547061+0000","last_undegraded":"2024-09-14T03:34:03.547061+0000","last_fullsized":"2024-09-14T03:34:03.547061+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:47:55.233617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478695+0000","last_change":"2024-09-14T03:34:02.515661+0000","last_active":"2024-09-14T03:34:04.478695+0000","last_peered":"2024-09-14T03:34:04.478695+0000","last_clean":"2024-09-14T03:34:04.478695+0000","last_became_active":"2024-09-14T03:34:02.515374+0000","last_became_peered":"2024-09-14T03:34:02.515374+0000","last_unstale":"2024-09-14T03:34:04.478695+0000","last_undegraded":"2024-09-14T03:34:04.478695+0000","last_fullsized":"2024-09-14T03:34:04.478695+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:06.066379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.516531+0000","last_change":"2024-09-14T03:33:59.400168+0000","last_active":"2024-09-14T03:34:04.516531+0000","last_peered":"2024-09-14T03:34:04.516531+0000","last_clean":"2024-09-14T03:34:04.516531+0000","last_became_active":"2024-09-14T03:33:59.399904+0000","last_became_peered":"2024-09-14T03:33:59.399904+0000","last_unstale":"2024-09-14T03:34:04.516531+0000","last_undegraded":"2024-09-14T03:34:04.516531+0000","last_fullsized":"2024-09-14T03:34:04.516531+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.387300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508444+0000","last_change":"2024-09-14T03:34:02.514049+0000","last_active":"2024-09-14T03:34:03.508444+0000","last_peered":"2024-09-14T03:34:03.508444+0000","last_clean":"2024-09-14T03:34:03.508444+0000","last_became_active":"2024-09-14T03:34:02.513309+0000","last_became_peered":"2024-09-14T03:34:02.513309+0000","last_unstale":"2024-09-14T03:34:03.508444+0000","last_undegraded":"2024-09-14T03:34:03.508444+0000","last_fullsized":"2024-09-14T03:34:03.508444+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:14.127299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.514634+0000","last_change":"2024-09-14T03:33:59.406006+0000","last_active":"2024-09-14T03:34:04.514634+0000","last_peered":"2024-09-14T03:34:04.514634+0000","last_clean":"2024-09-14T03:34:04.514634+0000","last_became_active":"2024-09-14T03:33:59.405514+0000","last_became_peered":"2024-09-14T03:33:59.405514+0000","last_unstale":"2024-09-14T03:34:04.514634+0000","last_undegraded":"2024-09-14T03:34:04.514634+0000","last_fullsized":"2024-09-14T03:34:04.514634+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:23:17.405355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478948+0000","last_change":"2024-09-14T03:33:59.399102+0000","last_active":"2024-09-14T03:34:04.478948+0000","last_peered":"2024-09-14T03:34:04.478948+0000","last_clean":"2024-09-14T03:34:04.478948+0000","last_became_active":"2024-09-14T03:33:59.398766+0000","last_became_peered":"2024-09-14T03:33:59.398766+0000","last_unstale":"2024-09-14T03:34:04.478948+0000","last_undegraded":"2024-09-14T03:34:04.478948+0000","last_fullsized":"2024-09-14T03:34:04.478948+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:08:01.084059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478803+0000","last_change":"2024-09-14T03:34:02.520106+0000","last_active":"2024-09-14T03:34:04.478803+0000","last_peered":"2024-09-14T03:34:04.478803+0000","last_clean":"2024-09-14T03:34:04.478803+0000","last_became_active":"2024-09-14T03:34:02.519825+0000","last_became_peered":"2024-09-14T03:34:02.519825+0000","last_unstale":"2024-09-14T03:34:04.478803+0000","last_undegraded":"2024-09-14T03:34:04.478803+0000","last_fullsized":"2024-09-14T03:34:04.478803+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:44.762587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478888+0000","last_change":"2024-09-14T03:33:59.406587+0000","last_active":"2024-09-14T03:34:04.478888+0000","last_peered":"2024-09-14T03:34:04.478888+0000","last_clean":"2024-09-14T03:34:04.478888+0000","last_became_active":"2024-09-14T03:33:59.405828+0000","last_became_peered":"2024-09-14T03:33:59.405828+0000","last_unstale":"2024-09-14T03:34:04.478888+0000","last_undegraded":"2024-09-14T03:34:04.478888+0000","last_fullsized":"2024-09-14T03:34:04.478888+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:57.857900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478585+0000","last_change":"2024-09-14T03:34:02.507004+0000","last_active":"2024-09-14T03:34:04.478585+0000","last_peered":"2024-09-14T03:34:04.478585+0000","last_clean":"2024-09-14T03:34:04.478585+0000","last_became_active":"2024-09-14T03:34:02.506311+0000","last_became_peered":"2024-09-14T03:34:02.506311+0000","last_unstale":"2024-09-14T03:34:04.478585+0000","last_undegraded":"2024-09-14T03:34:04.478585+0000","last_fullsized":"2024-09-14T03:34:04.478585+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:33.038366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547596+0000","last_change":"2024-09-14T03:33:59.402116+0000","last_active":"2024-09-14T03:34:03.547596+0000","last_peered":"2024-09-14T03:34:03.547596+0000","last_clean":"2024-09-14T03:34:03.547596+0000","last_became_active":"2024-09-14T03:33:59.400274+0000","last_became_peered":"2024-09-14T03:33:59.400274+0000","last_unstale":"2024-09-14T03:34:03.547596+0000","last_undegraded":"2024-09-14T03:34:03.547596+0000","last_fullsized":"2024-09-14T03:34:03.547596+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:46.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478905+0000","last_change":"2024-09-14T03:34:02.520187+0000","last_active":"2024-09-14T03:34:04.478905+0000","last_peered":"2024-09-14T03:34:04.478905+0000","last_clean":"2024-09-14T03:34:04.478905+0000","last_became_active":"2024-09-14T03:34:02.519909+0000","last_became_peered":"2024-09-14T03:34:02.519909+0000","last_unstale":"2024-09-14T03:34:04.478905+0000","last_undegraded":"2024-09-14T03:34:04.478905+0000","last_fullsized":"2024-09-14T03:34:04.478905+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:32.588302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479053+0000","last_change":"2024-09-14T03:33:59.402069+0000","last_active":"2024-09-14T03:34:04.479053+0000","last_peered":"2024-09-14T03:34:04.479053+0000","last_clean":"2024-09-14T03:34:04.479053+0000","last_became_active":"2024-09-14T03:33:59.401698+0000","last_became_peered":"2024-09-14T03:33:59.401698+0000","last_unstale":"2024-09-14T03:34:04.479053+0000","last_undegraded":"2024-09-14T03:34:04.479053+0000","last_fullsized":"2024-09-14T03:34:04.479053+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:16:48.483610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508760+0000","last_change":"2024-09-14T03:34:02.515495+0000","last_active":"2024-09-14T03:34:03.508760+0000","last_peered":"2024-09-14T03:34:03.508760+0000","last_clean":"2024-09-14T03:34:03.508760+0000","last_became_active":"2024-09-14T03:34:02.515256+0000","last_became_peered":"2024-09-14T03:34:02.515256+0000","last_unstale":"2024-09-14T03:34:03.508760+0000","last_undegraded":"2024-09-14T03:34:03.508760+0000","last_fullsized":"2024-09-14T03:34:03.508760+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:02.392960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.520648+0000","last_change":"2024-09-14T03:33:59.398566+0000","last_active":"2024-09-14T03:34:04.520648+0000","last_peered":"2024-09-14T03:34:04.520648+0000","last_clean":"2024-09-14T03:34:04.520648+0000","last_became_active":"2024-09-14T03:33:59.398281+0000","last_became_peered":"2024-09-14T03:33:59.398281+0000","last_unstale":"2024-09-14T03:34:04.520648+0000","last_undegraded":"2024-09-14T03:34:04.520648+0000","last_fullsized":"2024-09-14T03:34:04.520648+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:42.503990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479665+0000","last_change":"2024-09-14T03:34:02.510787+0000","last_active":"2024-09-14T03:34:04.479665+0000","last_peered":"2024-09-14T03:34:04.479665+0000","last_clean":"2024-09-14T03:34:04.479665+0000","last_became_active":"2024-09-14T03:34:02.510117+0000","last_became_peered":"2024-09-14T03:34:02.510117+0000","last_unstale":"2024-09-14T03:34:04.479665+0000","last_undegraded":"2024-09-14T03:34:04.479665+0000","last_fullsized":"2024-09-14T03:34:04.479665+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:13:53.616471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"58'6","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.522253+0000","last_change":"2024-09-14T03:33:59.401007+0000","last_active":"2024-09-14T03:34:04.522253+0000","last_peered":"2024-09-14T03:34:04.522253+0000","last_clean":"2024-09-14T03:34:04.522253+0000","last_became_active":"2024-09-14T03:33:59.400453+0000","last_became_peered":"2024-09-14T03:33:59.400453+0000","last_unstale":"2024-09-14T03:34:04.522253+0000","last_undegraded":"2024-09-14T03:34:04.522253+0000","last_fullsized":"2024-09-14T03:34:04.522253+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:32.633267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508331+0000","last_change":"2024-09-14T03:34:02.511714+0000","last_active":"2024-09-14T03:34:03.508331+0000","last_peered":"2024-09-14T03:34:03.508331+0000","last_clean":"2024-09-14T03:34:03.508331+0000","last_became_active":"2024-09-14T03:34:02.511328+0000","last_became_peered":"2024-09-14T03:34:02.511328+0000","last_unstale":"2024-09-14T03:34:03.508331+0000","last_undegraded":"2024-09-14T03:34:03.508331+0000","last_fullsized":"2024-09-14T03:34:03.508331+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:00:06.445335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.523143+0000","last_change":"2024-09-14T03:33:59.396984+0000","last_active":"2024-09-14T03:34:04.523143+0000","last_peered":"2024-09-14T03:34:04.523143+0000","last_clean":"2024-09-14T03:34:04.523143+0000","last_became_active":"2024-09-14T03:33:59.395835+0000","last_became_peered":"2024-09-14T03:33:59.395835+0000","last_unstale":"2024-09-14T03:34:04.523143+0000","last_undegraded":"2024-09-14T03:34:04.523143+0000","last_fullsized":"2024-09-14T03:34:04.523143+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:57.889158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508491+0000","last_change":"2024-09-14T03:34:02.521192+0000","last_active":"2024-09-14T03:34:03.508491+0000","last_peered":"2024-09-14T03:34:03.508491+0000","last_clean":"2024-09-14T03:34:03.508491+0000","last_became_active":"2024-09-14T03:34:02.520704+0000","last_became_peered":"2024-09-14T03:34:02.520704+0000","last_unstale":"2024-09-14T03:34:03.508491+0000","last_undegraded":"2024-09-14T03:34:03.508491+0000","last_fullsized":"2024-09-14T03:34:03.508491+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:37:13.469199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479662+0000","last_change":"2024-09-14T03:33:59.406737+0000","last_active":"2024-09-14T03:34:04.479662+0000","last_peered":"2024-09-14T03:34:04.479662+0000","last_clean":"2024-09-14T03:34:04.479662+0000","last_became_active":"2024-09-14T03:33:59.405882+0000","last_became_peered":"2024-09-14T03:33:59.405882+0000","last_unstale":"2024-09-14T03:34:04.479662+0000","last_undegraded":"2024-09-14T03:34:04.479662+0000","last_fullsized":"2024-09-14T03:34:04.479662+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:58.379549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508116+0000","last_change":"2024-09-14T03:34:02.517299+0000","last_active":"2024-09-14T03:34:03.508116+0000","last_peered":"2024-09-14T03:34:03.508116+0000","last_clean":"2024-09-14T03:34:03.508116+0000","last_became_active":"2024-09-14T03:34:02.516007+0000","last_became_peered":"2024-09-14T03:34:02.516007+0000","last_unstale":"2024-09-14T03:34:03.508116+0000","last_undegraded":"2024-09-14T03:34:03.508116+0000","last_fullsized":"2024-09-14T03:34:03.508116+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:49.173576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547149+0000","last_change":"2024-09-14T03:33:59.401544+0000","last_active":"2024-09-14T03:34:03.547149+0000","last_peered":"2024-09-14T03:34:03.547149+0000","last_clean":"2024-09-14T03:34:03.547149+0000","last_became_active":"2024-09-14T03:33:59.400073+0000","last_became_peered":"2024-09-14T03:33:59.400073+0000","last_unstale":"2024-09-14T03:34:03.547149+0000","last_undegraded":"2024-09-14T03:34:03.547149+0000","last_fullsized":"2024-09-14T03:34:03.547149+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:24.935055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547180+0000","last_change":"2024-09-14T03:34:02.516939+0000","last_active":"2024-09-14T03:34:03.547180+0000","last_peered":"2024-09-14T03:34:03.547180+0000","last_clean":"2024-09-14T03:34:03.547180+0000","last_became_active":"2024-09-14T03:34:02.516657+0000","last_became_peered":"2024-09-14T03:34:02.516657+0000","last_unstale":"2024-09-14T03:34:03.547180+0000","last_undegraded":"2024-09-14T03:34:03.547180+0000","last_fullsized":"2024-09-14T03:34:03.547180+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:17.423269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.523860+0000","last_change":"2024-09-14T03:33:59.400570+0000","last_active":"2024-09-14T03:34:04.523860+0000","last_peered":"2024-09-14T03:34:04.523860+0000","last_clean":"2024-09-14T03:34:04.523860+0000","last_became_active":"2024-09-14T03:33:59.400323+0000","last_became_peered":"2024-09-14T03:33:59.400323+0000","last_unstale":"2024-09-14T03:34:04.523860+0000","last_undegraded":"2024-09-14T03:34:04.523860+0000","last_fullsized":"2024-09-14T03:34:04.523860+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:39:46.484991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507971+0000","last_change":"2024-09-14T03:34:02.521031+0000","last_active":"2024-09-14T03:34:03.507971+0000","last_peered":"2024-09-14T03:34:03.507971+0000","last_clean":"2024-09-14T03:34:03.507971+0000","last_became_active":"2024-09-14T03:34:02.519973+0000","last_became_peered":"2024-09-14T03:34:02.519973+0000","last_unstale":"2024-09-14T03:34:03.507971+0000","last_undegraded":"2024-09-14T03:34:03.507971+0000","last_fullsized":"2024-09-14T03:34:03.507971+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:20.217955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.510003+0000","last_change":"2024-09-14T03:33:59.399709+0000","last_active":"2024-09-14T03:34:04.510003+0000","last_peered":"2024-09-14T03:34:04.510003+0000","last_clean":"2024-09-14T03:34:04.510003+0000","last_became_active":"2024-09-14T03:33:59.398533+0000","last_became_peered":"2024-09-14T03:33:59.398533+0000","last_unstale":"2024-09-14T03:34:04.510003+0000","last_undegraded":"2024-09-14T03:34:04.510003+0000","last_fullsized":"2024-09-14T03:34:04.510003+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:12:43.525175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508711+0000","last_change":"2024-09-14T03:34:02.520629+0000","last_active":"2024-09-14T03:34:03.508711+0000","last_peered":"2024-09-14T03:34:03.508711+0000","last_clean":"2024-09-14T03:34:03.508711+0000","last_became_active":"2024-09-14T03:34:02.520365+0000","last_became_peered":"2024-09-14T03:34:02.520365+0000","last_unstale":"2024-09-14T03:34:03.508711+0000","last_undegraded":"2024-09-14T03:34:03.508711+0000","last_fullsized":"2024-09-14T03:34:03.508711+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:29.519783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.523660+0000","last_change":"2024-09-14T03:33:59.404858+0000","last_active":"2024-09-14T03:34:04.523660+0000","last_peered":"2024-09-14T03:34:04.523660+0000","last_clean":"2024-09-14T03:34:04.523660+0000","last_became_active":"2024-09-14T03:33:59.404261+0000","last_became_peered":"2024-09-14T03:33:59.404261+0000","last_unstale":"2024-09-14T03:34:04.523660+0000","last_undegraded":"2024-09-14T03:34:04.523660+0000","last_fullsized":"2024-09-14T03:34:04.523660+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:24:38.834576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508776+0000","last_change":"2024-09-14T03:34:02.516816+0000","last_active":"2024-09-14T03:34:03.508776+0000","last_peered":"2024-09-14T03:34:03.508776+0000","last_clean":"2024-09-14T03:34:03.508776+0000","last_became_active":"2024-09-14T03:34:02.516527+0000","last_became_peered":"2024-09-14T03:34:02.516527+0000","last_unstale":"2024-09-14T03:34:03.508776+0000","last_undegraded":"2024-09-14T03:34:03.508776+0000","last_fullsized":"2024-09-14T03:34:03.508776+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:35.627973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.509308+0000","last_change":"2024-09-14T03:33:59.395532+0000","last_active":"2024-09-14T03:34:04.509308+0000","last_peered":"2024-09-14T03:34:04.509308+0000","last_clean":"2024-09-14T03:34:04.509308+0000","last_became_active":"2024-09-14T03:33:59.395255+0000","last_became_peered":"2024-09-14T03:33:59.395255+0000","last_unstale":"2024-09-14T03:34:04.509308+0000","last_undegraded":"2024-09-14T03:34:04.509308+0000","last_fullsized":"2024-09-14T03:34:04.509308+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:51.477913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479309+0000","last_change":"2024-09-14T03:34:02.520744+0000","last_active":"2024-09-14T03:34:04.479309+0000","last_peered":"2024-09-14T03:34:04.479309+0000","last_clean":"2024-09-14T03:34:04.479309+0000","last_became_active":"2024-09-14T03:34:02.520420+0000","last_became_peered":"2024-09-14T03:34:02.520420+0000","last_unstale":"2024-09-14T03:34:04.479309+0000","last_undegraded":"2024-09-14T03:34:04.479309+0000","last_fullsized":"2024-09-14T03:34:04.479309+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:32:43.573943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507667+0000","last_change":"2024-09-14T03:33:59.399299+0000","last_active":"2024-09-14T03:34:03.507667+0000","last_peered":"2024-09-14T03:34:03.507667+0000","last_clean":"2024-09-14T03:34:03.507667+0000","last_became_active":"2024-09-14T03:33:59.399008+0000","last_became_peered":"2024-09-14T03:33:59.399008+0000","last_unstale":"2024-09-14T03:34:03.507667+0000","last_undegraded":"2024-09-14T03:34:03.507667+0000","last_fullsized":"2024-09-14T03:34:03.507667+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:16:14.935721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508396+0000","last_change":"2024-09-14T03:34:02.521443+0000","last_active":"2024-09-14T03:34:03.508396+0000","last_peered":"2024-09-14T03:34:03.508396+0000","last_clean":"2024-09-14T03:34:03.508396+0000","last_became_active":"2024-09-14T03:34:02.519164+0000","last_became_peered":"2024-09-14T03:34:02.519164+0000","last_unstale":"2024-09-14T03:34:03.508396+0000","last_undegraded":"2024-09-14T03:34:03.508396+0000","last_fullsized":"2024-09-14T03:34:03.508396+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:28.199628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":23,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508242+0000","last_change":"2024-09-14T03:33:36.972974+0000","last_active":"2024-09-14T03:34:03.508242+0000","last_peered":"2024-09-14T03:34:03.508242+0000","last_clean":"2024-09-14T03:34:03.508242+0000","last_became_active":"2024-09-14T03:33:36.967695+0000","last_became_peered":"2024-09-14T03:33:36.967695+0000","last_unstale":"2024-09-14T03:34:03.508242+0000","last_undegraded":"2024-09-14T03:34:03.508242+0000","last_fullsized":"2024-09-14T03:34:03.508242+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_clean_scrub_stamp":"2024-09-14T03:31:39.719769+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:28:24.258573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478853+0000","last_change":"2024-09-14T03:33:59.406222+0000","last_active":"2024-09-14T03:34:04.478853+0000","last_peered":"2024-09-14T03:34:04.478853+0000","last_clean":"2024-09-14T03:34:04.478853+0000","last_became_active":"2024-09-14T03:33:59.405473+0000","last_became_peered":"2024-09-14T03:33:59.405473+0000","last_unstale":"2024-09-14T03:34:04.478853+0000","last_undegraded":"2024-09-14T03:34:04.478853+0000","last_fullsized":"2024-09-14T03:34:04.478853+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:26.326984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479087+0000","last_change":"2024-09-14T03:34:02.512416+0000","last_active":"2024-09-14T03:34:04.479087+0000","last_peered":"2024-09-14T03:34:04.479087+0000","last_clean":"2024-09-14T03:34:04.479087+0000","last_became_active":"2024-09-14T03:34:02.512135+0000","last_became_peered":"2024-09-14T03:34:02.512135+0000","last_unstale":"2024-09-14T03:34:04.479087+0000","last_undegraded":"2024-09-14T03:34:04.479087+0000","last_fullsized":"2024-09-14T03:34:04.479087+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:49.847882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508093+0000","last_change":"2024-09-14T03:33:59.409150+0000","last_active":"2024-09-14T03:34:03.508093+0000","last_peered":"2024-09-14T03:34:03.508093+0000","last_clean":"2024-09-14T03:34:03.508093+0000","last_became_active":"2024-09-14T03:33:59.408622+0000","last_became_peered":"2024-09-14T03:33:59.408622+0000","last_unstale":"2024-09-14T03:34:03.508093+0000","last_undegraded":"2024-09-14T03:34:03.508093+0000","last_fullsized":"2024-09-14T03:34:03.508093+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:11:03.508216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508080+0000","last_change":"2024-09-14T03:34:02.517068+0000","last_active":"2024-09-14T03:34:03.508080+0000","last_peered":"2024-09-14T03:34:03.508080+0000","last_clean":"2024-09-14T03:34:03.508080+0000","last_became_active":"2024-09-14T03:34:02.510158+0000","last_became_peered":"2024-09-14T03:34:02.510158+0000","last_unstale":"2024-09-14T03:34:03.508080+0000","last_undegraded":"2024-09-14T03:34:03.508080+0000","last_fullsized":"2024-09-14T03:34:03.508080+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:22.709122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.507560+0000","last_change":"2024-09-14T03:33:59.399987+0000","last_active":"2024-09-14T03:34:03.507560+0000","last_peered":"2024-09-14T03:34:03.507560+0000","last_clean":"2024-09-14T03:34:03.507560+0000","last_became_active":"2024-09-14T03:33:59.399698+0000","last_became_peered":"2024-09-14T03:33:59.399698+0000","last_unstale":"2024-09-14T03:34:03.507560+0000","last_undegraded":"2024-09-14T03:34:03.507560+0000","last_fullsized":"2024-09-14T03:34:03.507560+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:58.806927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478614+0000","last_change":"2024-09-14T03:34:02.515266+0000","last_active":"2024-09-14T03:34:04.478614+0000","last_peered":"2024-09-14T03:34:04.478614+0000","last_clean":"2024-09-14T03:34:04.478614+0000","last_became_active":"2024-09-14T03:34:02.515012+0000","last_became_peered":"2024-09-14T03:34:02.515012+0000","last_unstale":"2024-09-14T03:34:04.478614+0000","last_undegraded":"2024-09-14T03:34:04.478614+0000","last_fullsized":"2024-09-14T03:34:04.478614+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:52:23.510282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479727+0000","last_change":"2024-09-14T03:33:59.399736+0000","last_active":"2024-09-14T03:34:04.479727+0000","last_peered":"2024-09-14T03:34:04.479727+0000","last_clean":"2024-09-14T03:34:04.479727+0000","last_became_active":"2024-09-14T03:33:59.399434+0000","last_became_peered":"2024-09-14T03:33:59.399434+0000","last_unstale":"2024-09-14T03:34:04.479727+0000","last_undegraded":"2024-09-14T03:34:04.479727+0000","last_fullsized":"2024-09-14T03:34:04.479727+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:12.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479707+0000","last_change":"2024-09-14T03:34:02.507742+0000","last_active":"2024-09-14T03:34:04.479707+0000","last_peered":"2024-09-14T03:34:04.479707+0000","last_clean":"2024-09-14T03:34:04.479707+0000","last_became_active":"2024-09-14T03:34:02.507452+0000","last_became_peered":"2024-09-14T03:34:02.507452+0000","last_unstale":"2024-09-14T03:34:04.479707+0000","last_undegraded":"2024-09-14T03:34:04.479707+0000","last_fullsized":"2024-09-14T03:34:04.479707+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:50.407482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478910+0000","last_change":"2024-09-14T03:33:59.400498+0000","last_active":"2024-09-14T03:34:04.478910+0000","last_peered":"2024-09-14T03:34:04.478910+0000","last_clean":"2024-09-14T03:34:04.478910+0000","last_became_active":"2024-09-14T03:33:59.400204+0000","last_became_peered":"2024-09-14T03:33:59.400204+0000","last_unstale":"2024-09-14T03:34:04.478910+0000","last_undegraded":"2024-09-14T03:34:04.478910+0000","last_fullsized":"2024-09-14T03:34:04.478910+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:07:48.735887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479622+0000","last_change":"2024-09-14T03:34:02.513933+0000","last_active":"2024-09-14T03:34:04.479622+0000","last_peered":"2024-09-14T03:34:04.479622+0000","last_clean":"2024-09-14T03:34:04.479622+0000","last_became_active":"2024-09-14T03:34:02.513689+0000","last_became_peered":"2024-09-14T03:34:02.513689+0000","last_unstale":"2024-09-14T03:34:04.479622+0000","last_undegraded":"2024-09-14T03:34:04.479622+0000","last_fullsized":"2024-09-14T03:34:04.479622+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:23.791135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508518+0000","last_change":"2024-09-14T03:33:59.406992+0000","last_active":"2024-09-14T03:34:03.508518+0000","last_peered":"2024-09-14T03:34:03.508518+0000","last_clean":"2024-09-14T03:34:03.508518+0000","last_became_active":"2024-09-14T03:33:59.406729+0000","last_became_peered":"2024-09-14T03:33:59.406729+0000","last_unstale":"2024-09-14T03:34:03.508518+0000","last_undegraded":"2024-09-14T03:34:03.508518+0000","last_fullsized":"2024-09-14T03:34:03.508518+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:20:58.830368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478822+0000","last_change":"2024-09-14T03:34:02.513000+0000","last_active":"2024-09-14T03:34:04.478822+0000","last_peered":"2024-09-14T03:34:04.478822+0000","last_clean":"2024-09-14T03:34:04.478822+0000","last_became_active":"2024-09-14T03:34:02.512679+0000","last_became_peered":"2024-09-14T03:34:02.512679+0000","last_unstale":"2024-09-14T03:34:04.478822+0000","last_undegraded":"2024-09-14T03:34:04.478822+0000","last_fullsized":"2024-09-14T03:34:04.478822+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:43:48.821069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.510278+0000","last_change":"2024-09-14T03:33:59.397629+0000","last_active":"2024-09-14T03:34:04.510278+0000","last_peered":"2024-09-14T03:34:04.510278+0000","last_clean":"2024-09-14T03:34:04.510278+0000","last_became_active":"2024-09-14T03:33:59.397377+0000","last_became_peered":"2024-09-14T03:33:59.397377+0000","last_unstale":"2024-09-14T03:34:04.510278+0000","last_undegraded":"2024-09-14T03:34:04.510278+0000","last_fullsized":"2024-09-14T03:34:04.510278+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:42:02.914679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479450+0000","last_change":"2024-09-14T03:34:02.517598+0000","last_active":"2024-09-14T03:34:04.479450+0000","last_peered":"2024-09-14T03:34:04.479450+0000","last_clean":"2024-09-14T03:34:04.479450+0000","last_became_active":"2024-09-14T03:34:02.517367+0000","last_became_peered":"2024-09-14T03:34:02.517367+0000","last_unstale":"2024-09-14T03:34:04.479450+0000","last_undegraded":"2024-09-14T03:34:04.479450+0000","last_fullsized":"2024-09-14T03:34:04.479450+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:44.727080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.520324+0000","last_change":"2024-09-14T03:33:59.405301+0000","last_active":"2024-09-14T03:34:04.520324+0000","last_peered":"2024-09-14T03:34:04.520324+0000","last_clean":"2024-09-14T03:34:04.520324+0000","last_became_active":"2024-09-14T03:33:59.404994+0000","last_became_peered":"2024-09-14T03:33:59.404994+0000","last_unstale":"2024-09-14T03:34:04.520324+0000","last_undegraded":"2024-09-14T03:34:04.520324+0000","last_fullsized":"2024-09-14T03:34:04.520324+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:52:19.415253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508018+0000","last_change":"2024-09-14T03:34:02.516570+0000","last_active":"2024-09-14T03:34:03.508018+0000","last_peered":"2024-09-14T03:34:03.508018+0000","last_clean":"2024-09-14T03:34:03.508018+0000","last_became_active":"2024-09-14T03:34:02.516212+0000","last_became_peered":"2024-09-14T03:34:02.516212+0000","last_unstale":"2024-09-14T03:34:03.508018+0000","last_undegraded":"2024-09-14T03:34:03.508018+0000","last_fullsized":"2024-09-14T03:34:03.508018+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:33:23.282986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508340+0000","last_change":"2024-09-14T03:33:59.408805+0000","last_active":"2024-09-14T03:34:03.508340+0000","last_peered":"2024-09-14T03:34:03.508340+0000","last_clean":"2024-09-14T03:34:03.508340+0000","last_became_active":"2024-09-14T03:33:59.408521+0000","last_became_peered":"2024-09-14T03:33:59.408521+0000","last_unstale":"2024-09-14T03:34:03.508340+0000","last_undegraded":"2024-09-14T03:34:03.508340+0000","last_fullsized":"2024-09-14T03:34:03.508340+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:20:00.847563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478731+0000","last_change":"2024-09-14T03:34:02.511411+0000","last_active":"2024-09-14T03:34:04.478731+0000","last_peered":"2024-09-14T03:34:04.478731+0000","last_clean":"2024-09-14T03:34:04.478731+0000","last_became_active":"2024-09-14T03:34:02.511131+0000","last_became_peered":"2024-09-14T03:34:02.511131+0000","last_unstale":"2024-09-14T03:34:04.478731+0000","last_undegraded":"2024-09-14T03:34:04.478731+0000","last_fullsized":"2024-09-14T03:34:04.478731+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:23.033860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479335+0000","last_change":"2024-09-14T03:33:59.400483+0000","last_active":"2024-09-14T03:34:04.479335+0000","last_peered":"2024-09-14T03:34:04.479335+0000","last_clean":"2024-09-14T03:34:04.479335+0000","last_became_active":"2024-09-14T03:33:59.400172+0000","last_became_peered":"2024-09-14T03:33:59.400172+0000","last_unstale":"2024-09-14T03:34:04.479335+0000","last_undegraded":"2024-09-14T03:34:04.479335+0000","last_fullsized":"2024-09-14T03:34:04.479335+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:40.756478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478905+0000","last_change":"2024-09-14T03:34:02.505712+0000","last_active":"2024-09-14T03:34:04.478905+0000","last_peered":"2024-09-14T03:34:04.478905+0000","last_clean":"2024-09-14T03:34:04.478905+0000","last_became_active":"2024-09-14T03:34:02.505460+0000","last_became_peered":"2024-09-14T03:34:02.505460+0000","last_unstale":"2024-09-14T03:34:04.478905+0000","last_undegraded":"2024-09-14T03:34:04.478905+0000","last_fullsized":"2024-09-14T03:34:04.478905+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:20.752579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479364+0000","last_change":"2024-09-14T03:33:59.405640+0000","last_active":"2024-09-14T03:34:04.479364+0000","last_peered":"2024-09-14T03:34:04.479364+0000","last_clean":"2024-09-14T03:34:04.479364+0000","last_became_active":"2024-09-14T03:33:59.405382+0000","last_became_peered":"2024-09-14T03:33:59.405382+0000","last_unstale":"2024-09-14T03:34:04.479364+0000","last_undegraded":"2024-09-14T03:34:04.479364+0000","last_fullsized":"2024-09-14T03:34:04.479364+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:36.350128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479036+0000","last_change":"2024-09-14T03:34:02.516581+0000","last_active":"2024-09-14T03:34:04.479036+0000","last_peered":"2024-09-14T03:34:04.479036+0000","last_clean":"2024-09-14T03:34:04.479036+0000","last_became_active":"2024-09-14T03:34:02.516335+0000","last_became_peered":"2024-09-14T03:34:02.516335+0000","last_unstale":"2024-09-14T03:34:04.479036+0000","last_undegraded":"2024-09-14T03:34:04.479036+0000","last_fullsized":"2024-09-14T03:34:04.479036+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:06.070599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.521521+0000","last_change":"2024-09-14T03:33:59.408573+0000","last_active":"2024-09-14T03:34:04.521521+0000","last_peered":"2024-09-14T03:34:04.521521+0000","last_clean":"2024-09-14T03:34:04.521521+0000","last_became_active":"2024-09-14T03:33:59.408264+0000","last_became_peered":"2024-09-14T03:33:59.408264+0000","last_unstale":"2024-09-14T03:34:04.521521+0000","last_undegraded":"2024-09-14T03:34:04.521521+0000","last_fullsized":"2024-09-14T03:34:04.521521+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:30.937449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.547274+0000","last_change":"2024-09-14T03:34:02.516408+0000","last_active":"2024-09-14T03:34:03.547274+0000","last_peered":"2024-09-14T03:34:03.547274+0000","last_clean":"2024-09-14T03:34:03.547274+0000","last_became_active":"2024-09-14T03:34:02.516119+0000","last_became_peered":"2024-09-14T03:34:02.516119+0000","last_unstale":"2024-09-14T03:34:03.547274+0000","last_undegraded":"2024-09-14T03:34:03.547274+0000","last_fullsized":"2024-09-14T03:34:03.547274+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:38:16.570920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479461+0000","last_change":"2024-09-14T03:34:02.512557+0000","last_active":"2024-09-14T03:34:04.479461+0000","last_peered":"2024-09-14T03:34:04.479461+0000","last_clean":"2024-09-14T03:34:04.479461+0000","last_became_active":"2024-09-14T03:34:02.512183+0000","last_became_peered":"2024-09-14T03:34:02.512183+0000","last_unstale":"2024-09-14T03:34:04.479461+0000","last_undegraded":"2024-09-14T03:34:04.479461+0000","last_fullsized":"2024-09-14T03:34:04.479461+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:05:14.073579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.522438+0000","last_change":"2024-09-14T03:33:59.406333+0000","last_active":"2024-09-14T03:34:04.522438+0000","last_peered":"2024-09-14T03:34:04.522438+0000","last_clean":"2024-09-14T03:34:04.522438+0000","last_became_active":"2024-09-14T03:33:59.406065+0000","last_became_peered":"2024-09-14T03:33:59.406065+0000","last_unstale":"2024-09-14T03:34:04.522438+0000","last_undegraded":"2024-09-14T03:34:04.522438+0000","last_fullsized":"2024-09-14T03:34:04.522438+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:04.009889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.479258+0000","last_change":"2024-09-14T03:34:02.508363+0000","last_active":"2024-09-14T03:34:04.479258+0000","last_peered":"2024-09-14T03:34:04.479258+0000","last_clean":"2024-09-14T03:34:04.479258+0000","last_became_active":"2024-09-14T03:34:02.508086+0000","last_became_peered":"2024-09-14T03:34:02.508086+0000","last_unstale":"2024-09-14T03:34:04.479258+0000","last_undegraded":"2024-09-14T03:34:04.479258+0000","last_fullsized":"2024-09-14T03:34:04.479258+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:47.194036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.519684+0000","last_change":"2024-09-14T03:33:59.406945+0000","last_active":"2024-09-14T03:34:04.519684+0000","last_peered":"2024-09-14T03:34:04.519684+0000","last_clean":"2024-09-14T03:34:04.519684+0000","last_became_active":"2024-09-14T03:33:59.406662+0000","last_became_peered":"2024-09-14T03:33:59.406662+0000","last_unstale":"2024-09-14T03:34:04.519684+0000","last_undegraded":"2024-09-14T03:34:04.519684+0000","last_fullsized":"2024-09-14T03:34:04.519684+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:46:42.749128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:03.508697+0000","last_change":"2024-09-14T03:34:02.510922+0000","last_active":"2024-09-14T03:34:03.508697+0000","last_peered":"2024-09-14T03:34:03.508697+0000","last_clean":"2024-09-14T03:34:03.508697+0000","last_became_active":"2024-09-14T03:34:02.510663+0000","last_became_peered":"2024-09-14T03:34:02.510663+0000","last_unstale":"2024-09-14T03:34:03.508697+0000","last_undegraded":"2024-09-14T03:34:03.508697+0000","last_fullsized":"2024-09-14T03:34:03.508697+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:00:02.779787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.517651+0000","last_change":"2024-09-14T03:33:59.398992+0000","last_active":"2024-09-14T03:34:04.517651+0000","last_peered":"2024-09-14T03:34:04.517651+0000","last_clean":"2024-09-14T03:34:04.517651+0000","last_became_active":"2024-09-14T03:33:59.398725+0000","last_became_peered":"2024-09-14T03:33:59.398725+0000","last_unstale":"2024-09-14T03:34:04.517651+0000","last_undegraded":"2024-09-14T03:34:04.517651+0000","last_fullsized":"2024-09-14T03:34:04.517651+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:50:20.745864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.478974+0000","last_change":"2024-09-14T03:34:02.514810+0000","last_active":"2024-09-14T03:34:04.478974+0000","last_peered":"2024-09-14T03:34:04.478974+0000","last_clean":"2024-09-14T03:34:04.478974+0000","last_became_active":"2024-09-14T03:34:02.514500+0000","last_became_peered":"2024-09-14T03:34:02.514500+0000","last_unstale":"2024-09-14T03:34:04.478974+0000","last_undegraded":"2024-09-14T03:34:04.478974+0000","last_fullsized":"2024-09-14T03:34:04.478974+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:20:21.110046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:34:04.521464+0000","last_change":"2024-09-14T03:33:59.397898+0000","last_active":"2024-09-14T03:34:04.521464+0000","last_peered":"2024-09-14T03:34:04.521464+0000","last_clean":"2024-09-14T03:34:04.521464+0000","last_became_active":"2024-09-14T03:33:59.396670+0000","last_became_peered":"2024-09-14T03:33:59.396670+0000","last_unstale":"2024-09-14T03:34:04.521464+0000","last_undegraded":"2024-09-14T03:34:04.521464+0000","last_fullsized":"2024-09-14T03:34:04.521464+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:46.652243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397515,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1175552,"data_stored":831572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42799999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63900000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]}]},{"osd":6,"up_from":43,"seq":184683593744,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298512,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446640,"statfs":{"total":95995035648,"available":95689359360,"internally_reserved":0,"allocated":1175552,"data_stored":832217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52300000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.442},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.33000000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.38800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.373}]}]},{"osd":5,"up_from":37,"seq":158913789973,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":581632,"data_stored":241695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.64400000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.497},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.436}]},{"osd":2,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.52900000000000003}]},{"osd":3,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Sep 14 03:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.69199999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67100000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986202,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":589824,"data_stored":241308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.63200000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.57599999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:33:47 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.48999999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.502}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68300000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149726,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":593920,"data_stored":242268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:34:08 2024","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.35399999999999998,"15min":0.35799999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.439,"5min":0.502,"15min":0.502},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.41899999999999998,"15min":0.42699999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.48299999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.311}]},{"osd":1,"last update":"Sat Sep 14 03:34:08 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.38600000000000001,"15min":0.39000000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.501,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.38900000000000001,"15min":0.39500000000000002},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.52200000000000002,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:34:08 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.48099999999999998,"5min":0.54600000000000004,"15min":0.54600000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.39800000000000002,"15min":0.40400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.46100000000000002,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.26400000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:33:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.57899999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.55900000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61299999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411363,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":560,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":573440,"data_stored":240612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:33:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.40999999999999998,"15min":0.40799999999999997},"min":{"1min":0.219,"5min":0.16300000000000001,"15min":0.16300000000000001},"max":{"1min":0.59599999999999997,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.435,"15min":0.432},"min":{"1min":0.27600000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.68999999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 14 03:33:43 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.38500000000000001,"15min":0.38100000000000001},"min":{"1min":0.28100000000000003,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.39700000000000002},"min":{"1min":0.29299999999999998,"5min":0.224,"15min":0.224},"max":{"1min":0.58399999999999996,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:34:03 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.443,"15min":0.442},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.28199999999999997}]},{"osd":4,"last update":"Sat Sep 14 03:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.374}]},{"osd":5,"last update":"Sat Sep 14 03:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 14 03:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.36099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39400000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574887,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":556,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":569344,"data_stored":239827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:33:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.32700000000000001,"15min":0.315},"min":{"1min":0.29499999999999998,"5min":0.214,"15min":0.214},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.307},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.33900000000000002,"15min":0.33100000000000002},"min":{"1min":0.312,"5min":0.214,"15min":0.214},"max":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39800000000000002,"15min":0.39900000000000002},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.52000000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.41899999999999998,"15min":0.41499999999999998},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 14 03:34:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.45800000000000002,"15min":0.46600000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.27400000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.438}]},{"osd":5,"last update":"Sat Sep 14 03:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.374},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.496}]},{"osd":6,"last update":"Sat Sep 14 03:34:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.752,"5min":0.752,"15min":0.752},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":4.9050000000000002,"5min":4.9050000000000002,"15min":4.9050000000000002},"last":0.42299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1156,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1183744,"data_stored":831696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.32000000000000001,"15min":0.311},"min":{"1min":0.224,"5min":0.192,"15min":0.192},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.33600000000000002,"15min":0.32600000000000001},"min":{"1min":0.29199999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.35599999999999998},"min":{"1min":0.28100000000000003,"5min":0.253,"15min":0.253},"max":{"1min":0.53400000000000003,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.379,"15min":0.373},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.318}]},{"osd":3,"last update":"Sat Sep 14 03:34:07 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41799999999999998,"15min":0.41899999999999998},"min":{"1min":0.29299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.439},{"interface":"front","average":{"1min":0.435,"5min":0.45400000000000001,"15min":0.45700000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.379}]},{"osd":4,"last update":"Sat Sep 14 03:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.58299999999999996}]},{"osd":5,"last update":"Sat Sep 14 03:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.624},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.74399999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:34:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.5449999999999999,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.60999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:34:27.112 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T03:34:27.113 INFO:tasks.cephadm:Adding prometheus.a on smithi195 2024-09-14T03:34:27.113 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi195=a' 2024-09-14T03:34:27.371 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:34:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:27 smithi110 ceph-mon[27365]: from='client.14505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:34:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:27 smithi110 ceph-mon[27365]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:27 smithi110 ceph-mon[35374]: from='client.14505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:34:28.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:27 smithi110 ceph-mon[35374]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:27 smithi195 ceph-mon[30695]: from='client.14505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:34:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:27 smithi195 ceph-mon[30695]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:29.129 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled prometheus update... 2024-09-14T03:34:29.742 DEBUG:teuthology.orchestra.run.smithi195:prometheus.a> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-14T03:34:29.745 INFO:tasks.cephadm:Adding node-exporter.a on smithi110 2024-09-14T03:34:29.745 INFO:tasks.cephadm:Adding node-exporter.b on smithi195 2024-09-14T03:34:29.745 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi110=a;smithi195=b' 2024-09-14T03:34:30.063 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:34:30.392 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:30 smithi195 ceph-mon[30695]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:30.392 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:30 smithi195 ceph-mon[30695]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:30.393 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:30 smithi195 ceph-mon[30695]: Saving service prometheus spec with placement smithi195=a;count:1 2024-09-14T03:34:30.393 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:30 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:30.393 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:30 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:34:30.393 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:30 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:34:30.393 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:30 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:34:30.393 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:30 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:30.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[27365]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:30.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[27365]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:30.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[27365]: Saving service prometheus spec with placement smithi195=a;count:1 2024-09-14T03:34:30.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:30.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:34:30.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:34:30.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[35374]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[35374]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[35374]: Saving service prometheus spec with placement smithi195=a;count:1 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:34:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:30 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:31 smithi110 ceph-mon[27365]: Deploying daemon prometheus.a on smithi195 2024-09-14T03:34:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:31 smithi110 ceph-mon[35374]: Deploying daemon prometheus.a on smithi195 2024-09-14T03:34:31.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:31 smithi195 ceph-mon[30695]: Deploying daemon prometheus.a on smithi195 2024-09-14T03:34:32.321 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled node-exporter update... 2024-09-14T03:34:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:32 smithi110 ceph-mon[27365]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:32.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:32 smithi110 ceph-mon[35374]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:32.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:32 smithi195 ceph-mon[30695]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:33.262 DEBUG:teuthology.orchestra.run.smithi110:node-exporter.a> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-14T03:34:33.265 DEBUG:teuthology.orchestra.run.smithi195:node-exporter.b> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-14T03:34:33.267 INFO:tasks.cephadm:Adding alertmanager.a on smithi110 2024-09-14T03:34:33.267 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi110=a' 2024-09-14T03:34:33.552 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:34:33.672 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:33 smithi195 ceph-mon[30695]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi110=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:33.672 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:33 smithi195 ceph-mon[30695]: Saving service node-exporter spec with placement smithi110=a;smithi195=b;count:2 2024-09-14T03:34:33.672 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:33.673 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-14T03:34:33.673 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]: dispatch 2024-09-14T03:34:33.673 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [0, 2]}]: dispatch 2024-09-14T03:34:33.673 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]: dispatch 2024-09-14T03:34:33.673 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:33 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2024-09-14T03:34:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[27365]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi110=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[27365]: Saving service node-exporter spec with placement smithi110=a;smithi195=b;count:2 2024-09-14T03:34:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:33.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-14T03:34:33.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]: dispatch 2024-09-14T03:34:33.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [0, 2]}]: dispatch 2024-09-14T03:34:33.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]: dispatch 2024-09-14T03:34:33.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2024-09-14T03:34:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[35374]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi110=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[35374]: Saving service node-exporter spec with placement smithi110=a;smithi195=b;count:2 2024-09-14T03:34:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:33.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-14T03:34:33.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]: dispatch 2024-09-14T03:34:33.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [0, 2]}]: dispatch 2024-09-14T03:34:33.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]: dispatch 2024-09-14T03:34:33.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:33 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2024-09-14T03:34:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:34 smithi195 ceph-mon[30695]: pgmap v198: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:34 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-14T03:34:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:34 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]': finished 2024-09-14T03:34:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:34 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [0, 2]}]': finished 2024-09-14T03:34:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:34 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]': finished 2024-09-14T03:34:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:34 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2024-09-14T03:34:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:34 smithi195 ceph-mon[30695]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:34:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[27365]: pgmap v198: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-14T03:34:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]': finished 2024-09-14T03:34:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [0, 2]}]': finished 2024-09-14T03:34:34.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]': finished 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[27365]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[35374]: pgmap v198: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]': finished 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [0, 2]}]': finished 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]': finished 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2024-09-14T03:34:34.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:34 smithi110 ceph-mon[35374]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:34:35.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:35 smithi195 ceph-mon[30695]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:34:35.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:35 smithi110 ceph-mon[27365]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:34:35.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:35 smithi110 ceph-mon[35374]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:34:36.030 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled alertmanager update... 2024-09-14T03:34:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:36 smithi110 ceph-mon[27365]: pgmap v201: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:36.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:36 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:36.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:36 smithi110 ceph-mon[35374]: pgmap v201: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:36.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:36 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:36.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:36 smithi195 ceph-mon[30695]: pgmap v201: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:36.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:36 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:37.599 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:37 smithi195 ceph-mon[30695]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:37.599 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:37 smithi195 ceph-mon[30695]: Saving service alertmanager spec with placement smithi110=a;count:1 2024-09-14T03:34:37.642 DEBUG:teuthology.orchestra.run.smithi110:alertmanager.a> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-14T03:34:37.645 INFO:tasks.cephadm:Adding grafana.a on smithi195 2024-09-14T03:34:37.645 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi195=a' 2024-09-14T03:34:37.669 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:37 smithi110 ceph-mon[35374]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:37.669 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:37 smithi110 ceph-mon[35374]: Saving service alertmanager spec with placement smithi110=a;count:1 2024-09-14T03:34:37.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:37 smithi110 ceph-mon[27365]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:37.670 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:37 smithi110 ceph-mon[27365]: Saving service alertmanager spec with placement smithi110=a;count:1 2024-09-14T03:34:38.154 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:34:38.638 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:38 smithi195 ceph-mon[30695]: pgmap v202: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:38.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:38 smithi110 ceph-mon[35374]: pgmap v202: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:38.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:38 smithi110 ceph-mon[27365]: pgmap v202: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:40.621 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:40 smithi195 ceph-mon[30695]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:40.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:40 smithi110 ceph-mon[35374]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:40.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:40 smithi110 ceph-mon[27365]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:41.629 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled grafana update... 2024-09-14T03:34:42.357 DEBUG:teuthology.orchestra.run.smithi195:grafana.a> sudo journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service 2024-09-14T03:34:42.359 INFO:tasks.cephadm:Setting up client nodes... 2024-09-14T03:34:42.360 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-14T03:34:42.614 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:42.615 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:42 smithi195 ceph-mon[30695]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:42.616 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:42 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:42.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:42 smithi110 ceph-mon[35374]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:42.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:42 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:42.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:42 smithi110 ceph-mon[27365]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:42.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:42 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:43.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:43 smithi110 ceph-mon[27365]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:43.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:43 smithi110 ceph-mon[27365]: Saving service grafana spec with placement smithi195=a;count:1 2024-09-14T03:34:43.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:43 smithi110 ceph-mon[35374]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:43.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:43 smithi110 ceph-mon[35374]: Saving service grafana spec with placement smithi195=a;count:1 2024-09-14T03:34:43.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:43 smithi195 ceph-mon[30695]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:34:43.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:43 smithi195 ceph-mon[30695]: Saving service grafana spec with placement smithi195=a;count:1 2024-09-14T03:34:43.992 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:43 smithi195 podman[58653]: 2024-09-14 03:34:43.962405692 +0000 UTC m=+0.138175823 container create d8f68feeab7845fedbd24e39f4db634d506f83f2be8d8d17b187599ea6ef45da (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:34:44.257 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 podman[58653]: 2024-09-14 03:34:44.206369151 +0000 UTC m=+0.382139279 container init d8f68feeab7845fedbd24e39f4db634d506f83f2be8d8d17b187599ea6ef45da (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:34:44.257 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 podman[58653]: 2024-09-14 03:34:44.209595292 +0000 UTC m=+0.385365430 container start d8f68feeab7845fedbd24e39f4db634d506f83f2be8d8d17b187599ea6ef45da (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:34:44.257 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.234Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:34:44.257 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.234Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:34:44.257 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.234Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi195 (none))" 2024-09-14T03:34:44.257 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.234Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:34:44.258 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.234Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:34:44.258 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.235Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:34:44.258 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.235Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:34:44.258 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.236Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:34:44.258 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.236Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:34:44.258 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.238Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.239Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.702µs 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.239Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.239Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.239Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=16.222µs wal_replay_duration=171.867µs wbl_replay_duration=138ns total_replay_duration=206.118µs 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.239Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.239Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.239Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.255Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.454716ms db_storage=677ns remote_storage=957ns web_handler=460ns query_engine=579ns scrape=466.787µs scrape_sd=17.733µs notify=802ns notify_sd=1.728µs rules=14.618462ms tracing=7.52µs 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.255Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:34:44.259 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:34:44.255Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:34:44.453 INFO:teuthology.orchestra.run.smithi110.stdout:[client.0] 2024-09-14T03:34:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: key = AQBUBOVmYhLMGhAAfyywT5yt4QaKG11QeP64Bw== 2024-09-14T03:34:44.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:44 smithi110 ceph-mon[35374]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:44.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:44 smithi110 ceph-mon[27365]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:44.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:44 smithi195 ceph-mon[30695]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:44.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 bash[58653]: d8f68feeab7845fedbd24e39f4db634d506f83f2be8d8d17b187599ea6ef45da 2024-09-14T03:34:44.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:34:44 smithi195 systemd[1]: Started Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:34:45.141 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:34:45.141 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-14T03:34:45.141 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-14T03:34:45.180 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-14T03:34:45.454 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:34:45.540 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.540 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.540 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:34:45.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3766551286' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:45.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3766551286' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:34:45.541 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ignoring --setuser ceph since I am not root 2024-09-14T03:34:45.541 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ignoring --setgroup ceph since I am not root 2024-09-14T03:34:45.541 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:45.538+0000 7f38818d3ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:34:45.662 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:34:45.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3766551286' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:45.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3766551286' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:34:45.663 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ignoring --setuser ceph since I am not root 2024-09-14T03:34:45.663 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ignoring --setgroup ceph since I am not root 2024-09-14T03:34:45.663 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mgr[27599]: -- 172.21.15.110:0/1416249188 <== mon.2 v2:172.21.15.110:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564bd3d80340 con 0x564bd3d8cc00 2024-09-14T03:34:45.664 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:45.534+0000 7fdfc8d4fec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:34:45.664 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:45.592+0000 7fdfc8d4fec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:34:45.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' 2024-09-14T03:34:45.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:34:45.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3766551286' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:45.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3766551286' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:34:45.932 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:45.595+0000 7f38818d3ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:34:45.932 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:45.661+0000 7f38818d3ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:34:45.932 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:45.784+0000 7f38818d3ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:34:45.932 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:45.918+0000 7f38818d3ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:34:45.933 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:45.661+0000 7fdfc8d4fec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:34:45.933 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:45.789+0000 7fdfc8d4fec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:34:46.193 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:45 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:45.930+0000 7fdfc8d4fec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:34:46.209 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:46.176+0000 7f38818d3ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:34:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:46 smithi110 ceph-mon[35374]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:34:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:46 smithi110 ceph-mon[35374]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-14T03:34:46.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:46 smithi110 ceph-mon[27365]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:34:46.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:46 smithi110 ceph-mon[27365]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-14T03:34:46.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:46 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:46.191+0000 7fdfc8d4fec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:34:46.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:46 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:46.253+0000 7fdfc8d4fec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:34:46.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:46 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:46.421+0000 7fdfc8d4fec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:34:46.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:46 smithi195 ceph-mon[30695]: from='mgr.14150 172.21.15.110:0/1730302828' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:34:46.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:46 smithi195 ceph-mon[30695]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-14T03:34:46.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:46.237+0000 7f38818d3ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:34:46.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:46.400+0000 7f38818d3ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:34:47.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:47.251+0000 7f38818d3ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:34:47.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:47.320+0000 7f38818d3ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:34:47.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:47 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:47.259+0000 7fdfc8d4fec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:34:47.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:47 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:47.322+0000 7fdfc8d4fec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:34:47.744 INFO:teuthology.orchestra.run.smithi195.stdout:[client.1] 2024-09-14T03:34:47.745 INFO:teuthology.orchestra.run.smithi195.stdout: key = AQBXBOVmgDkTLBAA46ESz0eX2DbgnqdULbx6jA== 2024-09-14T03:34:48.047 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:47 smithi195 ceph-mon[30695]: from='client.? 172.21.15.195:0/2470237168' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:48.047 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:47 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:48.047 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:47 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:34:48.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:47 smithi110 ceph-mon[35374]: from='client.? 172.21.15.195:0/2470237168' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:48.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:47 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:48.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:47 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:34:48.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:47 smithi110 ceph-mon[27365]: from='client.? 172.21.15.195:0/2470237168' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:48.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:47 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:34:48.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:47 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:34:48.145 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.031+0000 7fdfc8d4fec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:34:48.145 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.142+0000 7fdfc8d4fec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:34:48.341 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.046+0000 7f38818d3ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:34:48.341 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.159+0000 7f38818d3ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:34:48.341 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.280+0000 7f38818d3ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:34:48.341 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.340+0000 7f38818d3ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:34:48.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.261+0000 7fdfc8d4fec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:34:48.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.320+0000 7fdfc8d4fec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:34:48.465 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-14T03:34:48.465 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-14T03:34:48.465 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-14T03:34:48.504 INFO:tasks.cephadm:Creating RBD pool 2024-09-14T03:34:48.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-14T03:34:48.598 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.469+0000 7f38818d3ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:34:48.598 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.539+0000 7f38818d3ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:34:48.719 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.445+0000 7fdfc8d4fec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:34:48.719 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.512+0000 7fdfc8d4fec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:34:48.719 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.573+0000 7fdfc8d4fec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:34:48.719 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.717+0000 7fdfc8d4fec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:34:48.758 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:48.858 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.597+0000 7f38818d3ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:34:48.859 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.730+0000 7f38818d3ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:34:48.859 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.792+0000 7f38818d3ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:34:48.859 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.857+0000 7f38818d3ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:34:48.990 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.788+0000 7fdfc8d4fec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:34:48.990 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.853+0000 7fdfc8d4fec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:34:48.990 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.935+0000 7fdfc8d4fec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:34:49.208 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.937+0000 7f38818d3ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:34:49.209 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:48.993+0000 7f38818d3ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:34:49.209 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:49.049+0000 7f38818d3ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:34:49.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:48 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:48.996+0000 7fdfc8d4fec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:34:49.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.055+0000 7fdfc8d4fec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:34:50.037 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:49.784+0000 7f38818d3ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:34:50.037 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: [14/Sep/2024:03:34:49] ENGINE Bus STARTING 2024-09-14T03:34:50.037 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: CherryPy Checker: 2024-09-14T03:34:50.037 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: The Application mounted at '' has an empty config. 2024-09-14T03:34:50.038 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: 2024-09-14T03:34:50.038 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: [14/Sep/2024:03:34:49] ENGINE Serving on http://:::9283 2024-09-14T03:34:50.038 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: [14/Sep/2024:03:34:49] ENGINE Bus STARTED 2024-09-14T03:34:50.038 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: Standby manager daemon x restarted 2024-09-14T03:34:50.039 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: Standby manager daemon x started 2024-09-14T03:34:50.039 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:34:50.039 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:34:50.039 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:34:50.039 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:34:50.039 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: Active manager daemon y restarted 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: Activating manager daemon y 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: mgrmap e17: y(active, starting, since 0.00796327s), standbys: x 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:34:50.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:34:50.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:34:50.042 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:49 smithi195 ceph-mon[30695]: Manager daemon y is now available 2024-09-14T03:34:50.060 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.807+0000 7fdfc8d4fec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:34:50.060 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:34:49] ENGINE Bus STARTING 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: CherryPy Checker: 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: The Application mounted at '' has an empty config. 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.913+0000 7fdf75725640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.913+0000 7fdf75725640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.913+0000 7fdf75725640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.913+0000 7fdf75725640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.913+0000 7fdf75725640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.915+0000 7fdf7872b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.915+0000 7fdf7872b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.915+0000 7fdf7872b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.061 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.915+0000 7fdf7872b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:49.915+0000 7fdf7872b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:34:50] ENGINE Serving on http://:::9283 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:34:50] ENGINE Bus STARTED 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: Standby manager daemon x restarted 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: Standby manager daemon x started 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:34:50.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: Active manager daemon y restarted 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: Activating manager daemon y 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: mgrmap e17: y(active, starting, since 0.00796327s), standbys: x 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:34:50.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[27365]: Manager daemon y is now available 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: Standby manager daemon x restarted 2024-09-14T03:34:50.064 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: Standby manager daemon x started 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2295858979' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: Active manager daemon y restarted 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: Activating manager daemon y 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: mgrmap e17: y(active, starting, since 0.00796327s), standbys: x 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:34:50.065 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:34:50.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:34:50.067 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:49 smithi110 ceph-mon[35374]: Manager daemon y is now available 2024-09-14T03:34:50.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:34:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:34:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:34:50.996 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'rbd' created 2024-09-14T03:34:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:34:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:34:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:34:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:34:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:50.938+0000 7fdf61afe640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:34:50] ENGINE Error in HTTPServer.serve 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: Traceback (most recent call last): 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._connections.run(self.expiration_interval) 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._run(expiration_interval) 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:34:51.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: s = self.context.wrap_socket( 2024-09-14T03:34:51.199 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:34:51.199 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: return self.sslsocket_class._create( 2024-09-14T03:34:51.199 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:34:51.199 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self.do_handshake() 2024-09-14T03:34:51.199 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:34:51.199 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: self._sslobj.do_handshake() 2024-09-14T03:34:51.199 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:34:51.199 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:34:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:34:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:50 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:34:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:50 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:50 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:34:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:50 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:34:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:50 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:34:51.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:50 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:34:51.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:50 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:34:51.997 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1408959026' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: [14/Sep/2024:03:34:50] ENGINE Bus STARTING 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: [14/Sep/2024:03:34:50] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: [14/Sep/2024:03:34:50] ENGINE Bus STARTED 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: [14/Sep/2024:03:34:50] ENGINE Error in HTTPServer.serve 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: Traceback (most recent call last): 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:34:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._run(expiration_interval) 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: s = self.context.wrap_socket( 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: return self.sslsocket_class._create( 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: self.do_handshake() 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._sslobj.do_handshake() 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: mgrmap e18: y(active, since 1.13999s), standbys: x 2024-09-14T03:34:52.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:52.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:52.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1408959026' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:34:52.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:51 smithi195 ceph-mon[30695]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:34:52.264 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:34:52.386 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1408959026' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:34:52.386 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: [14/Sep/2024:03:34:50] ENGINE Bus STARTING 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: [14/Sep/2024:03:34:50] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: [14/Sep/2024:03:34:50] ENGINE Bus STARTED 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: [14/Sep/2024:03:34:50] ENGINE Error in HTTPServer.serve 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:34:52.387 INFO:journalctl@ceph.mon.c.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: self.do_handshake() 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: mgrmap e18: y(active, since 1.13999s), standbys: x 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1408959026' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:34:52.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[35374]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1408959026' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: [14/Sep/2024:03:34:50] ENGINE Bus STARTING 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: [14/Sep/2024:03:34:50] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: [14/Sep/2024:03:34:50] ENGINE Bus STARTED 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: [14/Sep/2024:03:34:50] ENGINE Error in HTTPServer.serve 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:34:52.389 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: self.do_handshake() 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:34:52.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: mgrmap e18: y(active, since 1.13999s), standbys: x 2024-09-14T03:34:52.391 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:52.391 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:52.391 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1408959026' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:34:52.391 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:51 smithi110 ceph-mon[27365]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:34:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: pgmap v5: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:34:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:34:53.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:52 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:34:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: pgmap v5: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: pgmap v5: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:34:53.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:53.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:34:53.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:52 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:34:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi195 to 2992M 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi110 to 2992M 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2689706214' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi195 to 2992M 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi110 to 2992M 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:34:54.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:34:54.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:34:54.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2689706214' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:34:54.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:53 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:34:54.252 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi195 to 2992M 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi110 to 2992M 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.253 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.254 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.254 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.254 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:34:54.254 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2689706214' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:34:54.254 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:53 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:34:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:54 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:34:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:54 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:34:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:54 smithi195 ceph-mon[30695]: pgmap v8: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:54 smithi195 ceph-mon[30695]: Deploying daemon alertmanager.a on smithi110 2024-09-14T03:34:55.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:54 smithi195 ceph-mon[30695]: mgrmap e20: y(active, since 5s), standbys: x 2024-09-14T03:34:55.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:54 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:34:55.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:54 smithi195 ceph-mon[30695]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:34:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:34:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:34:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[35374]: pgmap v8: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[35374]: Deploying daemon alertmanager.a on smithi110 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[35374]: mgrmap e20: y(active, since 5s), standbys: x 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[35374]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[27365]: pgmap v8: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[27365]: Deploying daemon alertmanager.a on smithi110 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[27365]: mgrmap e20: y(active, since 5s), standbys: x 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:34:55.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:54 smithi110 ceph-mon[27365]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:34:56.833 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:56 smithi110 ceph-mon[27365]: pgmap v10: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:56.833 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:56 smithi110 ceph-mon[27365]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:34:56.834 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:56 smithi110 ceph-mon[27365]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:34:56.834 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:56 smithi110 ceph-mon[27365]: Cluster is now healthy 2024-09-14T03:34:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:56 smithi110 ceph-mon[35374]: pgmap v10: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:56 smithi110 ceph-mon[35374]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:34:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:56 smithi110 ceph-mon[35374]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:34:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:56 smithi110 ceph-mon[35374]: Cluster is now healthy 2024-09-14T03:34:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:56 smithi195 ceph-mon[30695]: pgmap v10: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:34:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:56 smithi195 ceph-mon[30695]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:34:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:56 smithi195 ceph-mon[30695]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:34:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:56 smithi195 ceph-mon[30695]: Cluster is now healthy 2024-09-14T03:34:57.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:57 smithi110 ceph-mon[35374]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:34:57.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:57 smithi110 ceph-mon[27365]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:34:58.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:57 smithi195 ceph-mon[30695]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:34:58.423 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-14T03:34:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:34:58 smithi110 ceph-mon[27365]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 212 B/s wr, 15 op/s 2024-09-14T03:34:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:34:58 smithi110 ceph-mon[35374]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 212 B/s wr, 15 op/s 2024-09-14T03:34:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:34:58 smithi195 ceph-mon[30695]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 212 B/s wr, 15 op/s 2024-09-14T03:34:59.278 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:00.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:35:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:35:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:35:01.000 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:00 smithi110 ceph-mon[27365]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 170 B/s wr, 12 op/s 2024-09-14T03:35:01.001 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:00 smithi110 ceph-mon[35374]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 170 B/s wr, 12 op/s 2024-09-14T03:35:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:00 smithi195 ceph-mon[30695]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 170 B/s wr, 12 op/s 2024-09-14T03:35:02.600 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:35:02.779 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:02 smithi110 podman[66808]: 2024-09-14 03:35:02.678358362 +0000 UTC m=+0.127163836 volume create 8461e20ed19acae1a42fe3c4f218cdbceb38d4825dc64e11600c88721552c124 2024-09-14T03:35:02.779 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:02 smithi110 podman[66808]: 2024-09-14 03:35:02.778657598 +0000 UTC m=+0.227463070 container create 730197a8cb7fab1ddcf8f8d60ede85030d594968e270e347aff3e4f8c1854f01 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:03.074 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:02 smithi110 ceph-mon[35374]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 292 B/s wr, 10 op/s 2024-09-14T03:35:03.074 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:02 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2782580952' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:35:03.074 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:02 smithi110 ceph-mon[27365]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 292 B/s wr, 10 op/s 2024-09-14T03:35:03.074 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:02 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2782580952' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:35:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:02 smithi195 ceph-mon[30695]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 292 B/s wr, 10 op/s 2024-09-14T03:35:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:02 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2782580952' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:35:03.383 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 podman[66808]: 2024-09-14 03:35:03.073360815 +0000 UTC m=+0.522166286 container init 730197a8cb7fab1ddcf8f8d60ede85030d594968e270e347aff3e4f8c1854f01 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 podman[66808]: 2024-09-14 03:35:03.076237459 +0000 UTC m=+0.525042930 container start 730197a8cb7fab1ddcf8f8d60ede85030d594968e270e347aff3e4f8c1854f01 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:03.088Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:03.088Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:03.089Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.110 port=9094 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:03.090Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:03.112Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:03.112Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:03.116Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:03.116Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 bash[66808]: 730197a8cb7fab1ddcf8f8d60ede85030d594968e270e347aff3e4f8c1854f01 2024-09-14T03:35:03.384 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:03 smithi110 systemd[1]: Started Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:35:03.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:35:03] ENGINE Bus STOPPING 2024-09-14T03:35:03.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:35:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:35:03.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:35:03] ENGINE Bus STOPPED 2024-09-14T03:35:03.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:35:03] ENGINE Bus STARTING 2024-09-14T03:35:03.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:35:03] ENGINE Serving on http://:::9283 2024-09-14T03:35:03.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:35:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:35:03] ENGINE Bus STARTED 2024-09-14T03:35:03.828 INFO:teuthology.orchestra.run.smithi110.stdout:[{"version":20,"timestamp":"2024-09-14T03:34:52.933377+0000","name":"","changes":[{"name":"osd/host:smithi110/osd_memory_target","previous_value":"4211087769","new_value":"3137345945"}]}] 2024-09-14T03:35:03.829 INFO:tasks.ceph_manager:config epoch is 20 2024-09-14T03:35:03.829 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:35:03.829 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:35:03.830 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-14T03:35:04.075 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:04.474 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:35:04.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:35:04.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[27365]: Deploying daemon grafana.a on smithi195 2024-09-14T03:35:04.476 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.476 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.476 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:35:04.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:35:04.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.478 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:04 smithi110 ceph-mon[35374]: Deploying daemon grafana.a on smithi195 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:04 smithi195 ceph-mon[30695]: Deploying daemon grafana.a on smithi195 2024-09-14T03:35:05.374 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:05.090Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000159807s 2024-09-14T03:35:05.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-mon[27365]: pgmap v16: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-14T03:35:05.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:05.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:05.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-mon[27365]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:35:05.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-mon[35374]: pgmap v16: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-14T03:35:05.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:05.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:05.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:05 smithi110 ceph-mon[35374]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:35:05.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:05 smithi195 ceph-mon[30695]: pgmap v16: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-14T03:35:05.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:05 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:05.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:05 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:05 smithi195 ceph-mon[30695]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:35:05.803 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:35:06.598 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:06 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3847003059' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:35:06.599 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:06 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3847003059' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:35:06.625 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":20,"flags":0,"active_gid":14526,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":1440218930},{"type":"v1","addr":"172.21.15.110:6801","nonce":1440218930}]},"active_addr":"172.21.15.110:6801/1440218930","active_change":"2024-09-14T03:34:49.809579+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24416,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":61,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":2779556305}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":3893271533}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":842858805}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":2963489706}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":1863905331}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":3780478344}]}]} 2024-09-14T03:35:06.629 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-14T03:35:06.630 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T03:35:06.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:35:06.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:06 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3847003059' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:35:06.878 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:07.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:07 smithi110 ceph-mon[35374]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 113 B/s wr, 0 op/s 2024-09-14T03:35:07.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:07 smithi110 ceph-mon[27365]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 113 B/s wr, 0 op/s 2024-09-14T03:35:07.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:07 smithi195 ceph-mon[30695]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 113 B/s wr, 0 op/s 2024-09-14T03:35:08.587 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:35:08.587 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":68,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:35:04.975662+0000","last_up_change":"2024-09-14T03:33:34.960002+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:33:57.860054+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:34:01.436541+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:34:50.939554+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6803","nonce":1137454602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6805","nonce":1137454602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6809","nonce":1137454602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6807","nonce":1137454602}]},"public_addr":"172.21.15.110:6803/1137454602","cluster_addr":"172.21.15.110:6805/1137454602","heartbeat_back_addr":"172.21.15.110:6809/1137454602","heartbeat_front_addr":"172.21.15.110:6807/1137454602","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6811","nonce":3479666487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6813","nonce":3479666487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6817","nonce":3479666487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6815","nonce":3479666487}]},"public_addr":"172.21.15.110:6811/3479666487","cluster_addr":"172.21.15.110:6813/3479666487","heartbeat_back_addr":"172.21.15.110:6817/3479666487","heartbeat_front_addr":"172.21.15.110:6815/3479666487","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6819","nonce":3061361173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6821","nonce":3061361173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6825","nonce":3061361173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6823","nonce":3061361173}]},"public_addr":"172.21.15.110:6819/3061361173","cluster_addr":"172.21.15.110:6821/3061361173","heartbeat_back_addr":"172.21.15.110:6825/3061361173","heartbeat_front_addr":"172.21.15.110:6823/3061361173","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6827","nonce":3810786960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6829","nonce":3810786960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6833","nonce":3810786960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6831","nonce":3810786960}]},"public_addr":"172.21.15.110:6827/3810786960","cluster_addr":"172.21.15.110:6829/3810786960","heartbeat_back_addr":"172.21.15.110:6833/3810786960","heartbeat_front_addr":"172.21.15.110:6831/3810786960","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6801","nonce":3499292350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6803","nonce":3499292350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6807","nonce":3499292350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6805","nonce":3499292350}]},"public_addr":"172.21.15.195:6801/3499292350","cluster_addr":"172.21.15.195:6803/3499292350","heartbeat_back_addr":"172.21.15.195:6807/3499292350","heartbeat_front_addr":"172.21.15.195:6805/3499292350","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6809","nonce":4151750662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6811","nonce":4151750662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6815","nonce":4151750662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6813","nonce":4151750662}]},"public_addr":"172.21.15.195:6809/4151750662","cluster_addr":"172.21.15.195:6811/4151750662","heartbeat_back_addr":"172.21.15.195:6815/4151750662","heartbeat_front_addr":"172.21.15.195:6813/4151750662","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6817","nonce":258227795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6819","nonce":258227795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6823","nonce":258227795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6821","nonce":258227795}]},"public_addr":"172.21.15.195:6817/258227795","cluster_addr":"172.21.15.195:6819/258227795","heartbeat_back_addr":"172.21.15.195:6823/258227795","heartbeat_front_addr":"172.21.15.195:6821/258227795","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6825","nonce":284932944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6827","nonce":284932944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6831","nonce":284932944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6829","nonce":284932944}]},"public_addr":"172.21.15.195:6825/284932944","cluster_addr":"172.21.15.195:6827/284932944","heartbeat_back_addr":"172.21.15.195:6831/284932944","heartbeat_front_addr":"172.21.15.195:6829/284932944","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.5","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.23","mappings":[{"from":0,"to":2}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/2214025689":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/4093221761":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2856593023":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/1769339162":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6800/1862102623":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6801/1862102623":"2024-09-15T03:34:49.809403+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:35:09.186 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T03:35:09.186 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:35:09.437 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:09.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:09 smithi110 ceph-mon[35374]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-14T03:35:09.558 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:09 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3867416743' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:35:09.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:09 smithi110 ceph-mon[27365]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-14T03:35:09.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:09 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3867416743' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:35:09.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:09 smithi195 ceph-mon[30695]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-14T03:35:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:09 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3867416743' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:35:10.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:35:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:35:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:35:11.065 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:35:11.066 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":68,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:35:04.975662+0000","last_up_change":"2024-09-14T03:33:34.960002+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:33:57.860054+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:34:01.436541+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:34:50.939554+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6803","nonce":1137454602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6805","nonce":1137454602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6809","nonce":1137454602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6807","nonce":1137454602}]},"public_addr":"172.21.15.110:6803/1137454602","cluster_addr":"172.21.15.110:6805/1137454602","heartbeat_back_addr":"172.21.15.110:6809/1137454602","heartbeat_front_addr":"172.21.15.110:6807/1137454602","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6811","nonce":3479666487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6813","nonce":3479666487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6817","nonce":3479666487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6815","nonce":3479666487}]},"public_addr":"172.21.15.110:6811/3479666487","cluster_addr":"172.21.15.110:6813/3479666487","heartbeat_back_addr":"172.21.15.110:6817/3479666487","heartbeat_front_addr":"172.21.15.110:6815/3479666487","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6819","nonce":3061361173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6821","nonce":3061361173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6825","nonce":3061361173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6823","nonce":3061361173}]},"public_addr":"172.21.15.110:6819/3061361173","cluster_addr":"172.21.15.110:6821/3061361173","heartbeat_back_addr":"172.21.15.110:6825/3061361173","heartbeat_front_addr":"172.21.15.110:6823/3061361173","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6827","nonce":3810786960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6829","nonce":3810786960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6833","nonce":3810786960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6831","nonce":3810786960}]},"public_addr":"172.21.15.110:6827/3810786960","cluster_addr":"172.21.15.110:6829/3810786960","heartbeat_back_addr":"172.21.15.110:6833/3810786960","heartbeat_front_addr":"172.21.15.110:6831/3810786960","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6801","nonce":3499292350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6803","nonce":3499292350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6807","nonce":3499292350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6805","nonce":3499292350}]},"public_addr":"172.21.15.195:6801/3499292350","cluster_addr":"172.21.15.195:6803/3499292350","heartbeat_back_addr":"172.21.15.195:6807/3499292350","heartbeat_front_addr":"172.21.15.195:6805/3499292350","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6809","nonce":4151750662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6811","nonce":4151750662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6815","nonce":4151750662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6813","nonce":4151750662}]},"public_addr":"172.21.15.195:6809/4151750662","cluster_addr":"172.21.15.195:6811/4151750662","heartbeat_back_addr":"172.21.15.195:6815/4151750662","heartbeat_front_addr":"172.21.15.195:6813/4151750662","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6817","nonce":258227795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6819","nonce":258227795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6823","nonce":258227795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6821","nonce":258227795}]},"public_addr":"172.21.15.195:6817/258227795","cluster_addr":"172.21.15.195:6819/258227795","heartbeat_back_addr":"172.21.15.195:6823/258227795","heartbeat_front_addr":"172.21.15.195:6821/258227795","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6825","nonce":284932944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6827","nonce":284932944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6831","nonce":284932944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6829","nonce":284932944}]},"public_addr":"172.21.15.195:6825/284932944","cluster_addr":"172.21.15.195:6827/284932944","heartbeat_back_addr":"172.21.15.195:6831/284932944","heartbeat_front_addr":"172.21.15.195:6829/284932944","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.5","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.23","mappings":[{"from":0,"to":2}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/2214025689":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/4093221761":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2856593023":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/1769339162":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6800/1862102623":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6801/1862102623":"2024-09-15T03:34:49.809403+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:35:11.515 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:11 smithi110 ceph-mon[35374]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-14T03:35:11.515 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:11 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/37498542' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:35:11.515 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:11 smithi110 ceph-mon[27365]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-14T03:35:11.515 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:11 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/37498542' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:35:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:11 smithi195 ceph-mon[30695]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-14T03:35:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:11 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/37498542' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:35:12.119 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T03:35:12.120 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T03:35:12.121 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T03:35:12.122 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T03:35:12.122 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T03:35:12.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T03:35:12.124 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T03:35:12.125 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T03:35:12.641 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:12.641 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:12.642 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:12.647 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:12.647 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:12.648 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:12.649 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:12.656 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:13.382 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:13 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:13.092Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002447378s 2024-09-14T03:35:13.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:13 smithi195 systemd[1]: Starting Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:35:13.591 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:13 smithi195 ceph-mon[30695]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:13.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:13 smithi110 ceph-mon[35374]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:13.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:13 smithi110 ceph-mon[27365]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:13.903 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:13 smithi195 podman[60630]: 2024-09-14 03:35:13.670386242 +0000 UTC m=+0.021868657 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-14T03:35:13.903 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:13 smithi195 podman[60630]: 2024-09-14 03:35:13.780405968 +0000 UTC m=+0.131888373 container create 67ea5066f839923cd0a167ee64ef4a838dad38f4406a421590eec7940724c983 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, vendor=Red Hat, Inc., distribution-scope=public, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-14T03:35:14.155 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:13 smithi195 podman[60630]: 2024-09-14 03:35:13.99117604 +0000 UTC m=+0.342658448 container init 67ea5066f839923cd0a167ee64ef4a838dad38f4406a421590eec7940724c983 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, distribution-scope=public, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.buildah.version=1.32.0, version=8.5, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, name=ubi8, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8) 2024-09-14T03:35:14.156 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:13 smithi195 podman[60630]: 2024-09-14 03:35:13.994148535 +0000 UTC m=+0.345630942 container start 67ea5066f839923cd0a167ee64ef4a838dad38f4406a421590eec7940724c983 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, name=ubi8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.buildah.version=1.32.0, description=Ceph Grafana Container, release=239.1651231664, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8) 2024-09-14T03:35:14.156 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 bash[60630]: 67ea5066f839923cd0a167ee64ef4a838dad38f4406a421590eec7940724c983 2024-09-14T03:35:14.156 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 systemd[1]: Started Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:35:14.455 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.154816884Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-14T03:35:14.455 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155076072Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-14T03:35:14.455 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155100347Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:35:14.455 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155114018Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155125398Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.15513618Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155146667Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155157435Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155168155Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155178706Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155188698Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155198784Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155209844Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=settings t=2024-09-14T03:35:14.155220707Z level=info msg="App mode production" 2024-09-14T03:35:14.456 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=sqlstore t=2024-09-14T03:35:14.155285533Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=sqlstore t=2024-09-14T03:35:14.155308955Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.156332269Z level=info msg="Starting DB migrations" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.156416131Z level=info msg="Executing migration" id="create migration_log table" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.157019105Z level=info msg="Executing migration" id="create user table" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.157431503Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.157882455Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.158418896Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.158837567Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.159192365Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.159865466Z level=info msg="Executing migration" id="create user table v2" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.160225389Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-14T03:35:14.457 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.160552589Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.161051651Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.161384451Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.161747741Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.162145199Z level=info msg="Executing migration" id="Update user table charset" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.162441856Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.162842139Z level=info msg="Executing migration" id="Add missing user data" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.163176623Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.163539088Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.16391931Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.164351032Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-14T03:35:14.458 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.165656478Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.166059848Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.166490783Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.166905057Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.167318388Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.167752368Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.168031327Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.168369491Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.1687607Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.169135872Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-14T03:35:14.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.1694864Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.170143735Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.170495811Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.170888213Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.171286608Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.171643721Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.172049908Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.172349459Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.17265741Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.172982266Z level=info msg="Executing migration" id="create star table" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.173313258Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.173748784Z level=info msg="Executing migration" id="create org table v1" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.174171576Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-14T03:35:14.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.174584298Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.175000875Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.17541165Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.175819164Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.176201123Z level=info msg="Executing migration" id="Update org table charset" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.176459789Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.176765008Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.177071243Z level=info msg="Executing migration" id="create dashboard table" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.177434076Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.177832378Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.178221219Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.178603891Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.179022496Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-14T03:35:14.462 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.179370968Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.180191814Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.180567609Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.181038978Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.181444471Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.181805533Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.182211889Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.182466631Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.182971082Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.183482536Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.183941433Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.184327686Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.184788884Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-14T03:35:14.463 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.185167731Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.18550628Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.185800516Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.186062007Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.186529161Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.187035199Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.187460469Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.187923344Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.188207302Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.188579104Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.188945757Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.18917596Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.189530895Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-14T03:35:14.464 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.189894757Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.190802423Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.191159806Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.191535732Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.191922115Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.192213207Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.19250284Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.192983315Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.193355448Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.193618218Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.193901386Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.194274233Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.194795902Z level=info msg="Executing migration" id="create data_source table" 2024-09-14T03:35:14.465 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.195214311Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.195597434Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.195985535Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.196311646Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.196651436Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.19757556Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.197972412Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.198320765Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.198704577Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-14T03:35:14.466 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.199032056Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.199491193Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.200039587Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.200301489Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.200551517Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.201060837Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.201315005Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.201571876Z level=info msg="Executing migration" id="Add uid column" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.202101924Z level=info msg="Executing migration" id="Update uid value" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.202365561Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.202773837Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.203169377Z level=info msg="Executing migration" id="create api_key table" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.20353878Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.203950764Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-14T03:35:14.467 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.204339658Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.204770358Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.205143424Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.205480689Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.205861252Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.20686454Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.207226986Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.207572878Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.207960604Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.208339916Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.208653597Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.20897985Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.209207586Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-14T03:35:14.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.20968696Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.21018944Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.210440277Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.210954176Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.211467255Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.211857858Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.212219063Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.21254886Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.212940353Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.213319181Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.213719198Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.213983886Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-14T03:35:14.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.214198996Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.214685902Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.215192251Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.215428018Z level=info msg="Executing migration" id="create quota table v1" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.215784232Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.21616168Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.216402242Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.216764663Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.217154132Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.217652483Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.217906399Z level=info msg="Executing migration" id="create session table" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.218284726Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.218506524Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-14T03:35:14.470 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.218769424Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.219116201Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.219445696Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.219662161Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.219911705Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.220127712Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.22034405Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.220697723Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.220948863Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.221451644Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.22172591Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.22225967Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.22280132Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-14T03:35:14.471 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.223065265Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.223429199Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.223857251Z level=info msg="Executing migration" id="create alert table v1" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.224253343Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.22467261Z level=info msg="Executing migration" id="add index alert state" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.225048824Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.225423532Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.225785846Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.226185572Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.226506075Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.227869499Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.228207904Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.228580078Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-14T03:35:14.472 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.228882453Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.229176848Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.229489601Z level=info msg="Executing migration" id="Add column is_default" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.230135503Z level=info msg="Executing migration" id="Add column frequency" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.230708681Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.231274072Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.23186078Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.232242621Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.232461548Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.232722624Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.233070851Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.233450479Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-14T03:35:14.473 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.233838361Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.234191207Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.234535391Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.23514903Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.235748991Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.236029122Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.23639734Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.236772291Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.237404289Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.237668085Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.238042305Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.238405744Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-14T03:35:14.476 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.238656664Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.239108875Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.23955202Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.239990719Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.240393232Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.240832626Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.241265167Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.241498173Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.24215108Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.242530319Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.243197915Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.243565441Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.243986236Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-14T03:35:14.477 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.244315662Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-14T03:35:14.478 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.245815714Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-14T03:35:14.478 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.246186848Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-14T03:35:14.478 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.246565195Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-14T03:35:14.478 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.246849175Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-14T03:35:14.478 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.247123425Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-14T03:35:14.478 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.247358205Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.247997681Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.248628967Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.249014647Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.249380755Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.24963199Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.250249836Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.250676518Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.250964694Z level=info msg="Executing migration" id="Move region to single row" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.251253137Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.251584613Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.251969778Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.252319029Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-14T03:35:14.479 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.252662192Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.253017824Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.253355364Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.253591466Z level=info msg="Executing migration" id="create test_data table" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.25402389Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.254368832Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.254790928Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.255165965Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.255403104Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.255661358Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.255918854Z level=info msg="Executing migration" id="create team table" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.256253337Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.256660479Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-14T03:35:14.480 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.257037495Z level=info msg="Executing migration" id="create team member table" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.25738151Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.25776607Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.258137755Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.258501109Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.259169567Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.25983697Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.260495574Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.260863331Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.261269155Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.261680652Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.262061304Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.262492968Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-14T03:35:14.481 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.262910347Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.26328555Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.2636658Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.264039379Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.264285241Z level=info msg="Executing migration" id="create tag table" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.26463332Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.26502301Z level=info msg="Executing migration" id="create login attempt table" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.26535981Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.265773553Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.266174612Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.267978766Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.26832086Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.268713381Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-14T03:35:14.482 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.269020524Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.269505597Z level=info msg="Executing migration" id="create user auth table" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.269879419Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.270258158Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.270481534Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.271170309Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.271865494Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.272541474Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.273246728Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.273672341Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.274368033Z level=info msg="Executing migration" id="create server_lock table" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.274719373Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.27513336Z level=info msg="Executing migration" id="create user auth token table" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.27556421Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-14T03:35:14.483 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.275970631Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.276348873Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.276754756Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.277485272Z level=info msg="Executing migration" id="create cache_data table" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.277860694Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.278267399Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.278651275Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.279040512Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.279264101Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.279491696Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.27989837Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.280283926Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.280689049Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-14T03:35:14.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.280993779Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.28136032Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.281761449Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.282209309Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.282620601Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.283381318Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.283794241Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.284075743Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.284418661Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.284837793Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.285236292Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.285474966Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.285884379Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-14T03:35:14.485 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.286277216Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.28665293Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.287048064Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.287800599Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.288190836Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.288571561Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.291917656Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.295222663Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.295662183Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.296062709Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.296867442Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.297259514Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.297662879Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.298063549Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-14T03:35:14.486 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.298453744Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.298735162Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.299503493Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.30027472Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.301095377Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.301482862Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.301889755Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.302667217Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.303471006Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.303890897Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-14T03:35:14.487 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.304668972Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.3054405Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.305792048Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.306215903Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.306622984Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.307034846Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.307272746Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.308090149Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.308897073Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.30969115Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.31048897Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-14T03:35:14.488 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.311305819Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.311594459Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.311972599Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.312775527Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.313058568Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.313902365Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.31430945Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.31512353Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.315492179Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.315930339Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.316748942Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.317120815Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.317527871Z level=info msg="Executing migration" id="create alert_image table" 2024-09-14T03:35:14.489 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.317904919Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.318305091Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.318548507Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.318942314Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.319287619Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.319368983Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.319587873Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.319924256Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.320281462Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.320409749Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.320532395Z level=info msg="alerts found to migrate" alerts=0 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.320917996Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-14T03:35:14.490 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.321354056Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.321773033Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.322144658Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.322543618Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.322978333Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.323240632Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.323546277Z level=info msg="Executing migration" id="create data_keys table" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.323942056Z level=info msg="Executing migration" id="create secrets table" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.324344919Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.328529492Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.329397146Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.329653809Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.333812215Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.33802895Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-14T03:35:14.491 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.33841146Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.338848092Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.339171115Z level=info msg="Executing migration" id="create permission table" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.339544041Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.339971975Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.340389112Z level=info msg="Executing migration" id="create role table" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.340800691Z level=info msg="Executing migration" id="add column display_name" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.341692538Z level=info msg="Executing migration" id="add column group_name" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.342568007Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.342988471Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.343400851Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.343826332Z level=info msg="Executing migration" id="create team role table" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.344193216Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-14T03:35:14.492 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.344600457Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.34505509Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.345468053Z level=info msg="Executing migration" id="create user role table" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.345846229Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.346257592Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.346666158Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.347077901Z level=info msg="Executing migration" id="create builtin role table" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.347534497Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.347956825Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.348367901Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.349341495Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.349809821Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.350225245Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-14T03:35:14.493 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.350592474Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.351013385Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.351370536Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.351789989Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.352727981Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.353101826Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.353511277Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.353807201Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.354196988Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.354619069Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.354980288Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.355230818Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.355526005Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-14T03:35:14.494 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.355896691Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.356345158Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.357291181Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.357566237Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.358003058Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.358412307Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.358840781Z level=info msg="Executing migration" id="add correlation config column" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.359790795Z level=info msg="Executing migration" id="create entity_events table" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.36019568Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.360591519Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.360693509Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.360957915Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.361044624Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:35:14.495 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.361261026Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.3615891Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.361997311Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.362414066Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.362837395Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.363247119Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.363604449Z level=info msg="Executing migration" id="Drop public config table" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.363974756Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.364332718Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.364733952Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.365330777Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.365763173Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.368674273Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-14T03:35:14.496 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.369647247Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.370611275Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.370896793Z level=info msg="Executing migration" id="add share column" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.371847206Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.37228313Z level=info msg="Executing migration" id="create file table" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.372710091Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.373140542Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.373635443Z level=info msg="Executing migration" id="create file_meta table" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.374011144Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.374432808Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.374720311Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.375104108Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.375361191Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.375778813Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-14T03:35:14.497 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.37676036Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.377085168Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.377481141Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.377835766Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.37811572Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.378405285Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.379372338Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.380346852Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.380745934Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.390424022Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.390883929Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.391307757Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.394734254Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-14T03:35:14.498 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.395048953Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-14T03:35:14.499 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.395346768Z level=info msg="Executing migration" id="create folder table" 2024-09-14T03:35:14.499 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.395726367Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-14T03:35:14.499 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.39615035Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-14T03:35:14.499 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.396566762Z level=info msg="Executing migration" id="Update folder title length" 2024-09-14T03:35:14.499 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.39684043Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-14T03:35:14.499 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=migrator t=2024-09-14T03:35:14.397244183Z level=info msg="migrations completed" performed=485 skipped=0 duration=240.847393ms 2024-09-14T03:35:14.499 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=sqlstore t=2024-09-14T03:35:14.397655056Z level=info msg="Created default organization" 2024-09-14T03:35:14.499 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=plugin.loader t=2024-09-14T03:35:14.454134975Z level=info msg="Plugin registered" pluginID=input 2024-09-14T03:35:14.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=plugin.loader t=2024-09-14T03:35:14.483527804Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-14T03:35:14.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=plugin.loader t=2024-09-14T03:35:14.483559165Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-14T03:35:14.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=secrets t=2024-09-14T03:35:14.483594805Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:35:14.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=query_data t=2024-09-14T03:35:14.485213655Z level=info msg="Query Service initialization" 2024-09-14T03:35:14.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=live.push_http t=2024-09-14T03:35:14.487837079Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:35:15.382 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-mon[30695]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:15.382 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.382 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.382 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.382 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.382 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-mon[30695]: Deploying daemon node-exporter.a on smithi110 2024-09-14T03:35:15.382 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.382 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=infra.usagestats.collector t=2024-09-14T03:35:15.207625374Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=provisioning.datasources t=2024-09-14T03:35:15.208011015Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=provisioning.datasources t=2024-09-14T03:35:15.241591586Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=provisioning.datasources t=2024-09-14T03:35:15.273076314Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=provisioning.alerting t=2024-09-14T03:35:15.294431177Z level=info msg="starting to provision alerting" 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=provisioning.alerting t=2024-09-14T03:35:15.294450697Z level=info msg="finished to provision alerting" 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=grafanaStorageLogger t=2024-09-14T03:35:15.294553617Z level=info msg="storage starting" 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=ngalert.state.manager t=2024-09-14T03:35:15.294714967Z level=info msg="Warming state cache for startup" 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=http.server t=2024-09-14T03:35:15.296604352Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=ngalert.state.manager t=2024-09-14T03:35:15.304873567Z level=info msg="State cache has been initialized" states=0 duration=10.157686ms 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=sqlstore.transactions t=2024-09-14T03:35:15.304898957Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:35:15.304912599Z level=info msg="starting MultiOrg Alertmanager" 2024-09-14T03:35:15.383 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=ticker t=2024-09-14T03:35:15.304932571Z level=info msg=starting first_tick=2024-09-14T03:35:20Z 2024-09-14T03:35:15.384 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=sqlstore.transactions t=2024-09-14T03:35:15.340548778Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:35:15.384 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=sqlstore.transactions t=2024-09-14T03:35:15.380956016Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:35:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[35374]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[35374]: Deploying daemon node-exporter.a on smithi110 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[27365]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[27365]: Deploying daemon node-exporter.a on smithi110 2024-09-14T03:35:15.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:15 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:15.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:35:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[60652]: logger=sqlstore.transactions t=2024-09-14T03:35:15.391479177Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-14T03:35:17.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:17 smithi110 ceph-mon[27365]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:17.335 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:17 smithi110 ceph-mon[35374]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:17.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:17 smithi195 ceph-mon[30695]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:19 smithi110 ceph-mon[27365]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:19 smithi110 ceph-mon[35374]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:19.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:19 smithi195 ceph-mon[30695]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:20.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:20.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:20.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:35:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:35:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:35:21.433 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:21 smithi110 ceph-mon[35374]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:21.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:21 smithi110 ceph-mon[27365]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:21 smithi195 ceph-mon[30695]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:22.066 INFO:teuthology.orchestra.run.smithi110.stdout:158913789985 2024-09-14T03:35:22.066 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T03:35:23.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:23 smithi110 ceph-mon[35374]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:23.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:23 smithi110 ceph-mon[27365]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:23.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:23 smithi195 ceph-mon[30695]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:24.335 INFO:teuthology.orchestra.run.smithi110.stdout:210453397528 2024-09-14T03:35:24.335 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T03:35:25.033 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:24 smithi110 systemd[1]: Starting Ceph node-exporter.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:35:25.056 INFO:teuthology.orchestra.run.smithi110.stdout:34359738422 2024-09-14T03:35:25.056 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T03:35:25.363 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:25 smithi110 ceph-mon[27365]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:25.364 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:25 smithi110 ceph-mon[35374]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:25.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:25 smithi195 ceph-mon[30695]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:26.042 INFO:teuthology.orchestra.run.smithi110.stdout:77309411375 2024-09-14T03:35:26.042 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T03:35:26.829 INFO:teuthology.orchestra.run.smithi110.stdout:55834574900 2024-09-14T03:35:26.829 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T03:35:27.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:27 smithi110 ceph-mon[35374]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:27.336 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:27 smithi110 ceph-mon[27365]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:27.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:27 smithi195 ceph-mon[30695]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:27.618 INFO:teuthology.orchestra.run.smithi110.stdout:133143986215 2024-09-14T03:35:27.618 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T03:35:28.042 INFO:teuthology.orchestra.run.smithi110.stdout:111669149737 2024-09-14T03:35:28.042 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T03:35:28.073 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:28.074 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:28.076 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:28.091 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:28.178 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:28.276 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:28 smithi110 bash[69079]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-14T03:35:28.946 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:28 smithi110 bash[69079]: Getting image source signatures 2024-09-14T03:35:28.946 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:28 smithi110 bash[69079]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-14T03:35:28.946 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:28 smithi110 bash[69079]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-14T03:35:28.946 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:28 smithi110 bash[69079]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-14T03:35:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:29 smithi110 ceph-mon[35374]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:29 smithi110 ceph-mon[27365]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:29.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:29 smithi195 ceph-mon[30695]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:29.590 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:30.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:35:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:35:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:35:30.844 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:30 smithi110 bash[69079]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-14T03:35:30.944 INFO:teuthology.orchestra.run.smithi110.stdout:184683593757 2024-09-14T03:35:30.945 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T03:35:31.160 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:30 smithi110 bash[69079]: Writing manifest to image destination 2024-09-14T03:35:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:31 smithi110 ceph-mon[35374]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:31 smithi110 ceph-mon[27365]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:31.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:31 smithi195 ceph-mon[30695]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:31.763 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:31 smithi110 podman[69079]: 2024-09-14 03:35:31.435512434 +0000 UTC m=+3.369258743 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-14T03:35:31.763 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:31 smithi110 podman[69079]: 2024-09-14 03:35:31.659275766 +0000 UTC m=+3.593022084 container create 986afca49027ee03b55ff104e680d54c34410fc4ff196bda718e48370db38706 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 podman[69079]: 2024-09-14 03:35:32.038756622 +0000 UTC m=+3.972502933 container init 986afca49027ee03b55ff104e680d54c34410fc4ff196bda718e48370db38706 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 podman[69079]: 2024-09-14 03:35:32.041699373 +0000 UTC m=+3.975445681 container start 986afca49027ee03b55ff104e680d54c34410fc4ff196bda718e48370db38706 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.045Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.045Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.046Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.046Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:35:32.115 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:35:32.116 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:35:32.116 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:35:32.116 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:35:32.116 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:35:32.116 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:35:32.116 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.047Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:35:32.117 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:35:32.118 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:35:32.119 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:35:32.119 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:35:32.119 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:35:32.119 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:35:32.119 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:35:32.119 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:35:32.119 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:35:32.120 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:35:32.120 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:35:32.120 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:35:32.120 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:35:32.120 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[69418]: ts=2024-09-14T03:35:32.048Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:35:32.120 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 bash[69079]: 986afca49027ee03b55ff104e680d54c34410fc4ff196bda718e48370db38706 2024-09-14T03:35:32.446 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:35:32 smithi110 systemd[1]: Started Ceph node-exporter.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:35:32.879 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:32 smithi195 systemd[1]: Starting Ceph node-exporter.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:35:32.884 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:33.179 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:33 smithi195 bash[60906]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-14T03:35:33.434 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:33 smithi195 ceph-mon[30695]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:33.435 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:33 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.435 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:33 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.435 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:33 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.435 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:33 smithi195 ceph-mon[30695]: Deploying daemon node-exporter.b on smithi195 2024-09-14T03:35:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[27365]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[27365]: Deploying daemon node-exporter.b on smithi195 2024-09-14T03:35:33.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[35374]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:33.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:33.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:33 smithi110 ceph-mon[35374]: Deploying daemon node-exporter.b on smithi195 2024-09-14T03:35:33.708 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:33 smithi195 bash[60906]: Getting image source signatures 2024-09-14T03:35:33.709 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:33 smithi195 bash[60906]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-14T03:35:33.709 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:33 smithi195 bash[60906]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-14T03:35:33.709 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:33 smithi195 bash[60906]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-14T03:35:35.182 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:34 smithi195 bash[60906]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-14T03:35:35.182 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 bash[60906]: Writing manifest to image destination 2024-09-14T03:35:35.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:35 smithi110 ceph-mon[27365]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:35.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:35 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:35.281 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:35 smithi110 ceph-mon[35374]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:35.281 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:35 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:35.433 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 podman[60906]: 2024-09-14 03:35:35.326362604 +0000 UTC m=+2.351069667 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-14T03:35:35.434 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-mon[30695]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:35.434 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:35.712 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 podman[60906]: 2024-09-14 03:35:35.433230021 +0000 UTC m=+2.457937083 container create 67a66167cf529ff4c272934e3f565bc9ae53e05553b7649b27abd028f8507dc0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:35:35.712 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 podman[60906]: 2024-09-14 03:35:35.660423621 +0000 UTC m=+2.685130690 container init 67a66167cf529ff4c272934e3f565bc9ae53e05553b7649b27abd028f8507dc0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:35:35.712 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 podman[60906]: 2024-09-14 03:35:35.663674021 +0000 UTC m=+2.688381084 container start 67a66167cf529ff4c272934e3f565bc9ae53e05553b7649b27abd028f8507dc0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:35:35.712 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.667Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-14T03:35:35.712 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.667Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-14T03:35:35.712 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.668Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:35:35.712 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.668Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.668Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.668Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:35:35.713 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:35:35.714 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:35:35.715 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:35:35.716 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:35:35.717 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[60982]: ts=2024-09-14T03:35:35.669Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:35:35.754 INFO:teuthology.orchestra.run.smithi110.stdout:210453397530 2024-09-14T03:35:35.979 INFO:teuthology.orchestra.run.smithi110.stdout:111669149740 2024-09-14T03:35:36.185 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 bash[60906]: 67a66167cf529ff4c272934e3f565bc9ae53e05553b7649b27abd028f8507dc0 2024-09-14T03:35:36.185 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:35:35 smithi195 systemd[1]: Started Ceph node-exporter.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:35:36.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/225189849' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:35:36.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:35:36.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1283775642' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:35:36.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/225189849' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:35:36.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:35:36.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:36 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1283775642' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:35:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:36 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/225189849' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:35:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:36 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:36 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:36 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:36 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:36 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:36.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:36 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:35:36.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:36 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1283775642' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:35:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:37 smithi110 ceph-mon[35374]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:37.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:37 smithi110 ceph-mon[27365]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:37.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:37 smithi195 ceph-mon[30695]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:37.977 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149737 got 111669149740 for osd.3 2024-09-14T03:35:37.977 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:38.743 INFO:teuthology.orchestra.run.smithi110.stdout:34359738426 2024-09-14T03:35:39.098 INFO:teuthology.orchestra.run.smithi110.stdout:55834574902 2024-09-14T03:35:39.277 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:39 smithi195 ceph-mon[30695]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:39.278 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:39 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2707916010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:35:39.278 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:39 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/255921159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:35:39.394 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:39 smithi110 ceph-mon[35374]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:39.394 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:39 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2707916010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:35:39.395 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:39 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/255921159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:35:39.395 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:39 smithi110 ceph-mon[27365]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:39.395 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:39 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2707916010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:35:39.395 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:39 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/255921159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:35:40.147 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574900 got 55834574902 for osd.1 2024-09-14T03:35:40.147 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:40.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:35:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:35:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:35:40.746 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:40 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:40.746 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:40 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:40.746 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:40 smithi110 ceph-mon[35374]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:40.746 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:40 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:40.746 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:40 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:40.746 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:40 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:40.746 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:40 smithi110 ceph-mon[27365]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:40.747 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:40 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:40.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:40 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:40.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:40 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:40.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:40 smithi195 ceph-mon[30695]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:40.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:40 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:41.086 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397528 got 210453397530 for osd.7 2024-09-14T03:35:41.087 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:41.675 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738422 got 34359738426 for osd.0 2024-09-14T03:35:41.675 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:42.640 INFO:teuthology.orchestra.run.smithi110.stdout:158913789989 2024-09-14T03:35:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:42 smithi110 ceph-mon[27365]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:42 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/730222450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:35:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:42 smithi110 ceph-mon[35374]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:42 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/730222450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:35:43.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:42 smithi195 ceph-mon[30695]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:43.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:42 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/730222450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:35:43.271 INFO:teuthology.orchestra.run.smithi110.stdout:133143986218 2024-09-14T03:35:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:43 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3238834602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:35:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:43 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3238834602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:35:44.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:43 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3238834602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:35:44.226 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789985 got 158913789989 for osd.5 2024-09-14T03:35:44.226 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:44.486 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:44.569 INFO:teuthology.orchestra.run.smithi110.stdout:77309411379 2024-09-14T03:35:44.898 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986215 got 133143986218 for osd.4 2024-09-14T03:35:44.898 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:44 smithi110 ceph-mon[27365]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:44 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2284345502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:35:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:44 smithi110 ceph-mon[35374]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:44 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2284345502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:35:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:44 smithi195 ceph-mon[30695]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:44 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2284345502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:35:46.142 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411375 got 77309411379 for osd.2 2024-09-14T03:35:46.143 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:46 smithi110 ceph-mon[27365]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:46 smithi110 ceph-mon[35374]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:46 smithi195 ceph-mon[30695]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:47.865 INFO:teuthology.orchestra.run.smithi110.stdout:184683593760 2024-09-14T03:35:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:47 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1742031649' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:35:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:47 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1742031649' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:35:48.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:47 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1742031649' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:35:48.540 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593757 got 184683593760 for osd.6 2024-09-14T03:35:48.541 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:48.541 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T03:35:48.541 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:35:49.164 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:48 smithi110 ceph-mon[27365]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:49.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:48 smithi110 ceph-mon[35374]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:48 smithi195 ceph-mon[30695]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:49 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:50.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:49 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:50.209 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:35:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:35:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:35:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:49 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:35:51.084 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:35:51.086 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2024-09-14T03:35:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:50 smithi110 ceph-mon[35374]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:50 smithi110 ceph-mon[27365]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:51.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:50 smithi195 ceph-mon[30695]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:51.949 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-09-14T03:35:49.833921+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386112,"kb_used_data":7264,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575104,"statfs":{"total":767960285184,"available":765516906496,"internally_reserved":0,"allocated":7438336,"data_stored":4275878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002401"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916845+0000","last_change":"2024-09-14T03:34:02.519367+0000","last_active":"2024-09-14T03:34:56.916845+0000","last_peered":"2024-09-14T03:34:56.916845+0000","last_clean":"2024-09-14T03:34:56.916845+0000","last_became_active":"2024-09-14T03:34:02.518530+0000","last_became_peered":"2024-09-14T03:34:02.518530+0000","last_unstale":"2024-09-14T03:34:56.916845+0000","last_undegraded":"2024-09-14T03:34:56.916845+0000","last_fullsized":"2024-09-14T03:34:56.916845+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:02:36.205002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837707+0000","last_change":"2024-09-14T03:34:02.515212+0000","last_active":"2024-09-14T03:34:56.837707+0000","last_peered":"2024-09-14T03:34:56.837707+0000","last_clean":"2024-09-14T03:34:56.837707+0000","last_became_active":"2024-09-14T03:34:02.514951+0000","last_became_peered":"2024-09-14T03:34:02.514951+0000","last_unstale":"2024-09-14T03:34:56.837707+0000","last_undegraded":"2024-09-14T03:34:56.837707+0000","last_fullsized":"2024-09-14T03:34:56.837707+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:30.742131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831163+0000","last_change":"2024-09-14T03:34:02.504777+0000","last_active":"2024-09-14T03:34:56.831163+0000","last_peered":"2024-09-14T03:34:56.831163+0000","last_clean":"2024-09-14T03:34:56.831163+0000","last_became_active":"2024-09-14T03:34:02.504550+0000","last_became_peered":"2024-09-14T03:34:02.504550+0000","last_unstale":"2024-09-14T03:34:56.831163+0000","last_undegraded":"2024-09-14T03:34:56.831163+0000","last_fullsized":"2024-09-14T03:34:56.831163+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:59.864492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839205+0000","last_change":"2024-09-14T03:34:02.521444+0000","last_active":"2024-09-14T03:34:56.839205+0000","last_peered":"2024-09-14T03:34:56.839205+0000","last_clean":"2024-09-14T03:34:56.839205+0000","last_became_active":"2024-09-14T03:34:02.521149+0000","last_became_peered":"2024-09-14T03:34:02.521149+0000","last_unstale":"2024-09-14T03:34:56.839205+0000","last_undegraded":"2024-09-14T03:34:56.839205+0000","last_fullsized":"2024-09-14T03:34:56.839205+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:10.004625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916561+0000","last_change":"2024-09-14T03:34:02.513133+0000","last_active":"2024-09-14T03:34:56.916561+0000","last_peered":"2024-09-14T03:34:56.916561+0000","last_clean":"2024-09-14T03:34:56.916561+0000","last_became_active":"2024-09-14T03:34:02.512580+0000","last_became_peered":"2024-09-14T03:34:02.512580+0000","last_unstale":"2024-09-14T03:34:56.916561+0000","last_undegraded":"2024-09-14T03:34:56.916561+0000","last_fullsized":"2024-09-14T03:34:56.916561+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:03.077106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839116+0000","last_change":"2024-09-14T03:34:02.508230+0000","last_active":"2024-09-14T03:34:56.839116+0000","last_peered":"2024-09-14T03:34:56.839116+0000","last_clean":"2024-09-14T03:34:56.839116+0000","last_became_active":"2024-09-14T03:34:02.507921+0000","last_became_peered":"2024-09-14T03:34:02.507921+0000","last_unstale":"2024-09-14T03:34:56.839116+0000","last_undegraded":"2024-09-14T03:34:56.839116+0000","last_fullsized":"2024-09-14T03:34:56.839116+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:37.418870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.915849+0000","last_change":"2024-09-14T03:34:02.515319+0000","last_active":"2024-09-14T03:34:56.915849+0000","last_peered":"2024-09-14T03:34:56.915849+0000","last_clean":"2024-09-14T03:34:56.915849+0000","last_became_active":"2024-09-14T03:34:02.515022+0000","last_became_peered":"2024-09-14T03:34:02.515022+0000","last_unstale":"2024-09-14T03:34:56.915849+0000","last_undegraded":"2024-09-14T03:34:56.915849+0000","last_fullsized":"2024-09-14T03:34:56.915849+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:51:51.089278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859702+0000","last_change":"2024-09-14T03:34:02.522349+0000","last_active":"2024-09-14T03:34:56.859702+0000","last_peered":"2024-09-14T03:34:56.859702+0000","last_clean":"2024-09-14T03:34:56.859702+0000","last_became_active":"2024-09-14T03:34:02.522063+0000","last_became_peered":"2024-09-14T03:34:02.522063+0000","last_unstale":"2024-09-14T03:34:56.859702+0000","last_undegraded":"2024-09-14T03:34:56.859702+0000","last_fullsized":"2024-09-14T03:34:56.859702+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:15.378220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917135+0000","last_change":"2024-09-14T03:34:02.512031+0000","last_active":"2024-09-14T03:34:56.917135+0000","last_peered":"2024-09-14T03:34:56.917135+0000","last_clean":"2024-09-14T03:34:56.917135+0000","last_became_active":"2024-09-14T03:34:02.511364+0000","last_became_peered":"2024-09-14T03:34:02.511364+0000","last_unstale":"2024-09-14T03:34:56.917135+0000","last_undegraded":"2024-09-14T03:34:56.917135+0000","last_fullsized":"2024-09-14T03:34:56.917135+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:54:15.711934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917254+0000","last_change":"2024-09-14T03:34:02.518932+0000","last_active":"2024-09-14T03:34:56.917254+0000","last_peered":"2024-09-14T03:34:56.917254+0000","last_clean":"2024-09-14T03:34:56.917254+0000","last_became_active":"2024-09-14T03:34:02.518575+0000","last_became_peered":"2024-09-14T03:34:02.518575+0000","last_unstale":"2024-09-14T03:34:56.917254+0000","last_undegraded":"2024-09-14T03:34:56.917254+0000","last_fullsized":"2024-09-14T03:34:56.917254+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:01:11.929183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917339+0000","last_change":"2024-09-14T03:34:02.517977+0000","last_active":"2024-09-14T03:34:56.917339+0000","last_peered":"2024-09-14T03:34:56.917339+0000","last_clean":"2024-09-14T03:34:56.917339+0000","last_became_active":"2024-09-14T03:34:02.517385+0000","last_became_peered":"2024-09-14T03:34:02.517385+0000","last_unstale":"2024-09-14T03:34:56.917339+0000","last_undegraded":"2024-09-14T03:34:56.917339+0000","last_fullsized":"2024-09-14T03:34:56.917339+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:50.900533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830573+0000","last_change":"2024-09-14T03:34:02.510949+0000","last_active":"2024-09-14T03:34:56.830573+0000","last_peered":"2024-09-14T03:34:56.830573+0000","last_clean":"2024-09-14T03:34:56.830573+0000","last_became_active":"2024-09-14T03:34:02.510519+0000","last_became_peered":"2024-09-14T03:34:02.510519+0000","last_unstale":"2024-09-14T03:34:56.830573+0000","last_undegraded":"2024-09-14T03:34:56.830573+0000","last_fullsized":"2024-09-14T03:34:56.830573+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:06:16.860908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859767+0000","last_change":"2024-09-14T03:34:02.512463+0000","last_active":"2024-09-14T03:34:56.859767+0000","last_peered":"2024-09-14T03:34:56.859767+0000","last_clean":"2024-09-14T03:34:56.859767+0000","last_became_active":"2024-09-14T03:34:02.512201+0000","last_became_peered":"2024-09-14T03:34:02.512201+0000","last_unstale":"2024-09-14T03:34:56.859767+0000","last_undegraded":"2024-09-14T03:34:56.859767+0000","last_fullsized":"2024-09-14T03:34:56.859767+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:46.716482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830640+0000","last_change":"2024-09-14T03:34:02.513215+0000","last_active":"2024-09-14T03:34:56.830640+0000","last_peered":"2024-09-14T03:34:56.830640+0000","last_clean":"2024-09-14T03:34:56.830640+0000","last_became_active":"2024-09-14T03:34:02.512773+0000","last_became_peered":"2024-09-14T03:34:02.512773+0000","last_unstale":"2024-09-14T03:34:56.830640+0000","last_undegraded":"2024-09-14T03:34:56.830640+0000","last_fullsized":"2024-09-14T03:34:56.830640+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:27:04.454480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.836868+0000","last_change":"2024-09-14T03:34:02.503785+0000","last_active":"2024-09-14T03:34:56.836868+0000","last_peered":"2024-09-14T03:34:56.836868+0000","last_clean":"2024-09-14T03:34:56.836868+0000","last_became_active":"2024-09-14T03:34:02.503212+0000","last_became_peered":"2024-09-14T03:34:02.503212+0000","last_unstale":"2024-09-14T03:34:56.836868+0000","last_undegraded":"2024-09-14T03:34:56.836868+0000","last_fullsized":"2024-09-14T03:34:56.836868+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:15.408242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":31,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.838932+0000","last_change":"2024-09-14T03:34:34.953866+0000","last_active":"2024-09-14T03:34:56.838932+0000","last_peered":"2024-09-14T03:34:56.838932+0000","last_clean":"2024-09-14T03:34:56.838932+0000","last_became_active":"2024-09-14T03:34:34.953477+0000","last_became_peered":"2024-09-14T03:34:34.953477+0000","last_unstale":"2024-09-14T03:34:56.838932+0000","last_undegraded":"2024-09-14T03:34:56.838932+0000","last_fullsized":"2024-09-14T03:34:56.838932+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:22.262447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839661+0000","last_change":"2024-09-14T03:34:02.507186+0000","last_active":"2024-09-14T03:34:56.839661+0000","last_peered":"2024-09-14T03:34:56.839661+0000","last_clean":"2024-09-14T03:34:56.839661+0000","last_became_active":"2024-09-14T03:34:02.506907+0000","last_became_peered":"2024-09-14T03:34:02.506907+0000","last_unstale":"2024-09-14T03:34:56.839661+0000","last_undegraded":"2024-09-14T03:34:56.839661+0000","last_fullsized":"2024-09-14T03:34:56.839661+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:10:47.570717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830684+0000","last_change":"2024-09-14T03:34:02.519104+0000","last_active":"2024-09-14T03:34:56.830684+0000","last_peered":"2024-09-14T03:34:56.830684+0000","last_clean":"2024-09-14T03:34:56.830684+0000","last_became_active":"2024-09-14T03:34:02.518795+0000","last_became_peered":"2024-09-14T03:34:02.518795+0000","last_unstale":"2024-09-14T03:34:56.830684+0000","last_undegraded":"2024-09-14T03:34:56.830684+0000","last_fullsized":"2024-09-14T03:34:56.830684+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:34:49.356603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916926+0000","last_change":"2024-09-14T03:34:02.521848+0000","last_active":"2024-09-14T03:34:56.916926+0000","last_peered":"2024-09-14T03:34:56.916926+0000","last_clean":"2024-09-14T03:34:56.916926+0000","last_became_active":"2024-09-14T03:34:02.521559+0000","last_became_peered":"2024-09-14T03:34:02.521559+0000","last_unstale":"2024-09-14T03:34:56.916926+0000","last_undegraded":"2024-09-14T03:34:56.916926+0000","last_fullsized":"2024-09-14T03:34:56.916926+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:21:09.742073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916978+0000","last_change":"2024-09-14T03:34:02.515094+0000","last_active":"2024-09-14T03:34:56.916978+0000","last_peered":"2024-09-14T03:34:56.916978+0000","last_clean":"2024-09-14T03:34:56.916978+0000","last_became_active":"2024-09-14T03:34:02.514291+0000","last_became_peered":"2024-09-14T03:34:02.514291+0000","last_unstale":"2024-09-14T03:34:56.916978+0000","last_undegraded":"2024-09-14T03:34:56.916978+0000","last_fullsized":"2024-09-14T03:34:56.916978+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:39:52.842240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831065+0000","last_change":"2024-09-14T03:34:02.510758+0000","last_active":"2024-09-14T03:34:56.831065+0000","last_peered":"2024-09-14T03:34:56.831065+0000","last_clean":"2024-09-14T03:34:56.831065+0000","last_became_active":"2024-09-14T03:34:02.510485+0000","last_became_peered":"2024-09-14T03:34:02.510485+0000","last_unstale":"2024-09-14T03:34:56.831065+0000","last_undegraded":"2024-09-14T03:34:56.831065+0000","last_fullsized":"2024-09-14T03:34:56.831065+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:26:25.102759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839132+0000","last_change":"2024-09-14T03:34:02.511696+0000","last_active":"2024-09-14T03:34:56.839132+0000","last_peered":"2024-09-14T03:34:56.839132+0000","last_clean":"2024-09-14T03:34:56.839132+0000","last_became_active":"2024-09-14T03:34:02.511289+0000","last_became_peered":"2024-09-14T03:34:02.511289+0000","last_unstale":"2024-09-14T03:34:56.839132+0000","last_undegraded":"2024-09-14T03:34:56.839132+0000","last_fullsized":"2024-09-14T03:34:56.839132+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:02:27.657439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859569+0000","last_change":"2024-09-14T03:34:02.517564+0000","last_active":"2024-09-14T03:34:56.859569+0000","last_peered":"2024-09-14T03:34:56.859569+0000","last_clean":"2024-09-14T03:34:56.859569+0000","last_became_active":"2024-09-14T03:34:02.517314+0000","last_became_peered":"2024-09-14T03:34:02.517314+0000","last_unstale":"2024-09-14T03:34:56.859569+0000","last_undegraded":"2024-09-14T03:34:56.859569+0000","last_fullsized":"2024-09-14T03:34:56.859569+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:08.360046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.838946+0000","last_change":"2024-09-14T03:34:02.513865+0000","last_active":"2024-09-14T03:34:56.838946+0000","last_peered":"2024-09-14T03:34:56.838946+0000","last_clean":"2024-09-14T03:34:56.838946+0000","last_became_active":"2024-09-14T03:34:02.513008+0000","last_became_peered":"2024-09-14T03:34:02.513008+0000","last_unstale":"2024-09-14T03:34:56.838946+0000","last_undegraded":"2024-09-14T03:34:56.838946+0000","last_fullsized":"2024-09-14T03:34:56.838946+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:18:37.581596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859955+0000","last_change":"2024-09-14T03:34:02.517481+0000","last_active":"2024-09-14T03:34:56.859955+0000","last_peered":"2024-09-14T03:34:56.859955+0000","last_clean":"2024-09-14T03:34:56.859955+0000","last_became_active":"2024-09-14T03:34:02.517252+0000","last_became_peered":"2024-09-14T03:34:02.517252+0000","last_unstale":"2024-09-14T03:34:56.859955+0000","last_undegraded":"2024-09-14T03:34:56.859955+0000","last_fullsized":"2024-09-14T03:34:56.859955+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:29:15.209238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917253+0000","last_change":"2024-09-14T03:34:02.521201+0000","last_active":"2024-09-14T03:34:56.917253+0000","last_peered":"2024-09-14T03:34:56.917253+0000","last_clean":"2024-09-14T03:34:56.917253+0000","last_became_active":"2024-09-14T03:34:02.520764+0000","last_became_peered":"2024-09-14T03:34:02.520764+0000","last_unstale":"2024-09-14T03:34:56.917253+0000","last_undegraded":"2024-09-14T03:34:56.917253+0000","last_fullsized":"2024-09-14T03:34:56.917253+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:39:40.688146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837307+0000","last_change":"2024-09-14T03:34:02.504156+0000","last_active":"2024-09-14T03:34:56.837307+0000","last_peered":"2024-09-14T03:34:56.837307+0000","last_clean":"2024-09-14T03:34:56.837307+0000","last_became_active":"2024-09-14T03:34:02.503906+0000","last_became_peered":"2024-09-14T03:34:02.503906+0000","last_unstale":"2024-09-14T03:34:56.837307+0000","last_undegraded":"2024-09-14T03:34:56.837307+0000","last_fullsized":"2024-09-14T03:34:56.837307+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:33:57.350618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":31,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859822+0000","last_change":"2024-09-14T03:34:34.330855+0000","last_active":"2024-09-14T03:34:56.859822+0000","last_peered":"2024-09-14T03:34:56.859822+0000","last_clean":"2024-09-14T03:34:56.859822+0000","last_became_active":"2024-09-14T03:34:34.330493+0000","last_became_peered":"2024-09-14T03:34:34.330493+0000","last_unstale":"2024-09-14T03:34:56.859822+0000","last_undegraded":"2024-09-14T03:34:56.859822+0000","last_fullsized":"2024-09-14T03:34:56.859822+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:06.278074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830387+0000","last_change":"2024-09-14T03:34:34.955446+0000","last_active":"2024-09-14T03:34:56.830387+0000","last_peered":"2024-09-14T03:34:56.830387+0000","last_clean":"2024-09-14T03:34:56.830387+0000","last_became_active":"2024-09-14T03:34:34.955095+0000","last_became_peered":"2024-09-14T03:34:34.955095+0000","last_unstale":"2024-09-14T03:34:56.830387+0000","last_undegraded":"2024-09-14T03:34:56.830387+0000","last_fullsized":"2024-09-14T03:34:56.830387+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:01.417881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839234+0000","last_change":"2024-09-14T03:34:02.513294+0000","last_active":"2024-09-14T03:34:56.839234+0000","last_peered":"2024-09-14T03:34:56.839234+0000","last_clean":"2024-09-14T03:34:56.839234+0000","last_became_active":"2024-09-14T03:34:02.512998+0000","last_became_peered":"2024-09-14T03:34:02.512998+0000","last_unstale":"2024-09-14T03:34:56.839234+0000","last_undegraded":"2024-09-14T03:34:56.839234+0000","last_fullsized":"2024-09-14T03:34:56.839234+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:54.262830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830045+0000","last_change":"2024-09-14T03:34:02.512389+0000","last_active":"2024-09-14T03:34:56.830045+0000","last_peered":"2024-09-14T03:34:56.830045+0000","last_clean":"2024-09-14T03:34:56.830045+0000","last_became_active":"2024-09-14T03:34:02.511955+0000","last_became_peered":"2024-09-14T03:34:02.511955+0000","last_unstale":"2024-09-14T03:34:56.830045+0000","last_undegraded":"2024-09-14T03:34:56.830045+0000","last_fullsized":"2024-09-14T03:34:56.830045+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:30:31.454464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.915824+0000","last_change":"2024-09-14T03:34:02.520047+0000","last_active":"2024-09-14T03:34:56.915824+0000","last_peered":"2024-09-14T03:34:56.915824+0000","last_clean":"2024-09-14T03:34:56.915824+0000","last_became_active":"2024-09-14T03:34:02.519764+0000","last_became_peered":"2024-09-14T03:34:02.519764+0000","last_unstale":"2024-09-14T03:34:56.915824+0000","last_undegraded":"2024-09-14T03:34:56.915824+0000","last_fullsized":"2024-09-14T03:34:56.915824+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:47:55.233617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830465+0000","last_change":"2024-09-14T03:34:02.515661+0000","last_active":"2024-09-14T03:34:56.830465+0000","last_peered":"2024-09-14T03:34:56.830465+0000","last_clean":"2024-09-14T03:34:56.830465+0000","last_became_active":"2024-09-14T03:34:02.515374+0000","last_became_peered":"2024-09-14T03:34:02.515374+0000","last_unstale":"2024-09-14T03:34:56.830465+0000","last_undegraded":"2024-09-14T03:34:56.830465+0000","last_fullsized":"2024-09-14T03:34:56.830465+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:06.066379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859940+0000","last_change":"2024-09-14T03:33:59.400168+0000","last_active":"2024-09-14T03:34:56.859940+0000","last_peered":"2024-09-14T03:34:56.859940+0000","last_clean":"2024-09-14T03:34:56.859940+0000","last_became_active":"2024-09-14T03:33:59.399904+0000","last_became_peered":"2024-09-14T03:33:59.399904+0000","last_unstale":"2024-09-14T03:34:56.859940+0000","last_undegraded":"2024-09-14T03:34:56.859940+0000","last_fullsized":"2024-09-14T03:34:56.859940+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.387300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831124+0000","last_change":"2024-09-14T03:34:02.514049+0000","last_active":"2024-09-14T03:34:56.831124+0000","last_peered":"2024-09-14T03:34:56.831124+0000","last_clean":"2024-09-14T03:34:56.831124+0000","last_became_active":"2024-09-14T03:34:02.513309+0000","last_became_peered":"2024-09-14T03:34:02.513309+0000","last_unstale":"2024-09-14T03:34:56.831124+0000","last_undegraded":"2024-09-14T03:34:56.831124+0000","last_fullsized":"2024-09-14T03:34:56.831124+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:14.127299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916208+0000","last_change":"2024-09-14T03:33:59.406006+0000","last_active":"2024-09-14T03:34:56.916208+0000","last_peered":"2024-09-14T03:34:56.916208+0000","last_clean":"2024-09-14T03:34:56.916208+0000","last_became_active":"2024-09-14T03:33:59.405514+0000","last_became_peered":"2024-09-14T03:33:59.405514+0000","last_unstale":"2024-09-14T03:34:56.916208+0000","last_undegraded":"2024-09-14T03:34:56.916208+0000","last_fullsized":"2024-09-14T03:34:56.916208+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:23:17.405355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859715+0000","last_change":"2024-09-14T03:33:59.399102+0000","last_active":"2024-09-14T03:34:56.859715+0000","last_peered":"2024-09-14T03:34:56.859715+0000","last_clean":"2024-09-14T03:34:56.859715+0000","last_became_active":"2024-09-14T03:33:59.398766+0000","last_became_peered":"2024-09-14T03:33:59.398766+0000","last_unstale":"2024-09-14T03:34:56.859715+0000","last_undegraded":"2024-09-14T03:34:56.859715+0000","last_fullsized":"2024-09-14T03:34:56.859715+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:08:01.084059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859784+0000","last_change":"2024-09-14T03:34:02.520106+0000","last_active":"2024-09-14T03:34:56.859784+0000","last_peered":"2024-09-14T03:34:56.859784+0000","last_clean":"2024-09-14T03:34:56.859784+0000","last_became_active":"2024-09-14T03:34:02.519825+0000","last_became_peered":"2024-09-14T03:34:02.519825+0000","last_unstale":"2024-09-14T03:34:56.859784+0000","last_undegraded":"2024-09-14T03:34:56.859784+0000","last_fullsized":"2024-09-14T03:34:56.859784+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:44.762587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839014+0000","last_change":"2024-09-14T03:33:59.406587+0000","last_active":"2024-09-14T03:34:56.839014+0000","last_peered":"2024-09-14T03:34:56.839014+0000","last_clean":"2024-09-14T03:34:56.839014+0000","last_became_active":"2024-09-14T03:33:59.405828+0000","last_became_peered":"2024-09-14T03:33:59.405828+0000","last_unstale":"2024-09-14T03:34:56.839014+0000","last_undegraded":"2024-09-14T03:34:56.839014+0000","last_fullsized":"2024-09-14T03:34:56.839014+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:57.857900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.836777+0000","last_change":"2024-09-14T03:34:02.507004+0000","last_active":"2024-09-14T03:34:56.836777+0000","last_peered":"2024-09-14T03:34:56.836777+0000","last_clean":"2024-09-14T03:34:56.836777+0000","last_became_active":"2024-09-14T03:34:02.506311+0000","last_became_peered":"2024-09-14T03:34:02.506311+0000","last_unstale":"2024-09-14T03:34:56.836777+0000","last_undegraded":"2024-09-14T03:34:56.836777+0000","last_fullsized":"2024-09-14T03:34:56.836777+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:33.038366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916191+0000","last_change":"2024-09-14T03:33:59.402116+0000","last_active":"2024-09-14T03:34:56.916191+0000","last_peered":"2024-09-14T03:34:56.916191+0000","last_clean":"2024-09-14T03:34:56.916191+0000","last_became_active":"2024-09-14T03:33:59.400274+0000","last_became_peered":"2024-09-14T03:33:59.400274+0000","last_unstale":"2024-09-14T03:34:56.916191+0000","last_undegraded":"2024-09-14T03:34:56.916191+0000","last_fullsized":"2024-09-14T03:34:56.916191+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:46.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839481+0000","last_change":"2024-09-14T03:34:02.520187+0000","last_active":"2024-09-14T03:34:56.839481+0000","last_peered":"2024-09-14T03:34:56.839481+0000","last_clean":"2024-09-14T03:34:56.839481+0000","last_became_active":"2024-09-14T03:34:02.519909+0000","last_became_peered":"2024-09-14T03:34:02.519909+0000","last_unstale":"2024-09-14T03:34:56.839481+0000","last_undegraded":"2024-09-14T03:34:56.839481+0000","last_fullsized":"2024-09-14T03:34:56.839481+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:32.588302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839588+0000","last_change":"2024-09-14T03:33:59.402069+0000","last_active":"2024-09-14T03:34:56.839588+0000","last_peered":"2024-09-14T03:34:56.839588+0000","last_clean":"2024-09-14T03:34:56.839588+0000","last_became_active":"2024-09-14T03:33:59.401698+0000","last_became_peered":"2024-09-14T03:33:59.401698+0000","last_unstale":"2024-09-14T03:34:56.839588+0000","last_undegraded":"2024-09-14T03:34:56.839588+0000","last_fullsized":"2024-09-14T03:34:56.839588+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:16:48.483610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831335+0000","last_change":"2024-09-14T03:34:02.515495+0000","last_active":"2024-09-14T03:34:56.831335+0000","last_peered":"2024-09-14T03:34:56.831335+0000","last_clean":"2024-09-14T03:34:56.831335+0000","last_became_active":"2024-09-14T03:34:02.515256+0000","last_became_peered":"2024-09-14T03:34:02.515256+0000","last_unstale":"2024-09-14T03:34:56.831335+0000","last_undegraded":"2024-09-14T03:34:56.831335+0000","last_fullsized":"2024-09-14T03:34:56.831335+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:02.392960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":27,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837428+0000","last_change":"2024-09-14T03:33:59.398566+0000","last_active":"2024-09-14T03:34:56.837428+0000","last_peered":"2024-09-14T03:34:56.837428+0000","last_clean":"2024-09-14T03:34:56.837428+0000","last_became_active":"2024-09-14T03:33:59.398281+0000","last_became_peered":"2024-09-14T03:33:59.398281+0000","last_unstale":"2024-09-14T03:34:56.837428+0000","last_undegraded":"2024-09-14T03:34:56.837428+0000","last_fullsized":"2024-09-14T03:34:56.837428+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:42.503990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830716+0000","last_change":"2024-09-14T03:34:34.953658+0000","last_active":"2024-09-14T03:34:56.830716+0000","last_peered":"2024-09-14T03:34:56.830716+0000","last_clean":"2024-09-14T03:34:56.830716+0000","last_became_active":"2024-09-14T03:34:34.953335+0000","last_became_peered":"2024-09-14T03:34:34.953335+0000","last_unstale":"2024-09-14T03:34:56.830716+0000","last_undegraded":"2024-09-14T03:34:56.830716+0000","last_fullsized":"2024-09-14T03:34:56.830716+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:40.796694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.14","version":"61'9","reported_seq":33,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917005+0000","last_change":"2024-09-14T03:33:59.401007+0000","last_active":"2024-09-14T03:34:56.917005+0000","last_peered":"2024-09-14T03:34:56.917005+0000","last_clean":"2024-09-14T03:34:56.917005+0000","last_became_active":"2024-09-14T03:33:59.400453+0000","last_became_peered":"2024-09-14T03:33:59.400453+0000","last_unstale":"2024-09-14T03:34:56.917005+0000","last_undegraded":"2024-09-14T03:34:56.917005+0000","last_fullsized":"2024-09-14T03:34:56.917005+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:32.633267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916702+0000","last_change":"2024-09-14T03:34:02.511714+0000","last_active":"2024-09-14T03:34:56.916702+0000","last_peered":"2024-09-14T03:34:56.916702+0000","last_clean":"2024-09-14T03:34:56.916702+0000","last_became_active":"2024-09-14T03:34:02.511328+0000","last_became_peered":"2024-09-14T03:34:02.511328+0000","last_unstale":"2024-09-14T03:34:56.916702+0000","last_undegraded":"2024-09-14T03:34:56.916702+0000","last_fullsized":"2024-09-14T03:34:56.916702+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:00:06.445335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831162+0000","last_change":"2024-09-14T03:33:59.396984+0000","last_active":"2024-09-14T03:34:56.831162+0000","last_peered":"2024-09-14T03:34:56.831162+0000","last_clean":"2024-09-14T03:34:56.831162+0000","last_became_active":"2024-09-14T03:33:59.395835+0000","last_became_peered":"2024-09-14T03:33:59.395835+0000","last_unstale":"2024-09-14T03:34:56.831162+0000","last_undegraded":"2024-09-14T03:34:56.831162+0000","last_fullsized":"2024-09-14T03:34:56.831162+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:57.889158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830709+0000","last_change":"2024-09-14T03:34:02.521192+0000","last_active":"2024-09-14T03:34:56.830709+0000","last_peered":"2024-09-14T03:34:56.830709+0000","last_clean":"2024-09-14T03:34:56.830709+0000","last_became_active":"2024-09-14T03:34:02.520704+0000","last_became_peered":"2024-09-14T03:34:02.520704+0000","last_unstale":"2024-09-14T03:34:56.830709+0000","last_undegraded":"2024-09-14T03:34:56.830709+0000","last_fullsized":"2024-09-14T03:34:56.830709+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:37:13.469199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839263+0000","last_change":"2024-09-14T03:33:59.406737+0000","last_active":"2024-09-14T03:34:56.839263+0000","last_peered":"2024-09-14T03:34:56.839263+0000","last_clean":"2024-09-14T03:34:56.839263+0000","last_became_active":"2024-09-14T03:33:59.405882+0000","last_became_peered":"2024-09-14T03:33:59.405882+0000","last_unstale":"2024-09-14T03:34:56.839263+0000","last_undegraded":"2024-09-14T03:34:56.839263+0000","last_fullsized":"2024-09-14T03:34:56.839263+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:58.379549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916562+0000","last_change":"2024-09-14T03:34:02.517299+0000","last_active":"2024-09-14T03:34:56.916562+0000","last_peered":"2024-09-14T03:34:56.916562+0000","last_clean":"2024-09-14T03:34:56.916562+0000","last_became_active":"2024-09-14T03:34:02.516007+0000","last_became_peered":"2024-09-14T03:34:02.516007+0000","last_unstale":"2024-09-14T03:34:56.916562+0000","last_undegraded":"2024-09-14T03:34:56.916562+0000","last_fullsized":"2024-09-14T03:34:56.916562+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:49.173576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916024+0000","last_change":"2024-09-14T03:33:59.401544+0000","last_active":"2024-09-14T03:34:56.916024+0000","last_peered":"2024-09-14T03:34:56.916024+0000","last_clean":"2024-09-14T03:34:56.916024+0000","last_became_active":"2024-09-14T03:33:59.400073+0000","last_became_peered":"2024-09-14T03:33:59.400073+0000","last_unstale":"2024-09-14T03:34:56.916024+0000","last_undegraded":"2024-09-14T03:34:56.916024+0000","last_fullsized":"2024-09-14T03:34:56.916024+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:24.935055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916207+0000","last_change":"2024-09-14T03:34:02.516939+0000","last_active":"2024-09-14T03:34:56.916207+0000","last_peered":"2024-09-14T03:34:56.916207+0000","last_clean":"2024-09-14T03:34:56.916207+0000","last_became_active":"2024-09-14T03:34:02.516657+0000","last_became_peered":"2024-09-14T03:34:02.516657+0000","last_unstale":"2024-09-14T03:34:56.916207+0000","last_undegraded":"2024-09-14T03:34:56.916207+0000","last_fullsized":"2024-09-14T03:34:56.916207+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:17.423269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":32,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830004+0000","last_change":"2024-09-14T03:33:59.400570+0000","last_active":"2024-09-14T03:34:56.830004+0000","last_peered":"2024-09-14T03:34:56.830004+0000","last_clean":"2024-09-14T03:34:56.830004+0000","last_became_active":"2024-09-14T03:33:59.400323+0000","last_became_peered":"2024-09-14T03:33:59.400323+0000","last_unstale":"2024-09-14T03:34:56.830004+0000","last_undegraded":"2024-09-14T03:34:56.830004+0000","last_fullsized":"2024-09-14T03:34:56.830004+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:39:46.484991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.915927+0000","last_change":"2024-09-14T03:34:02.521031+0000","last_active":"2024-09-14T03:34:56.915927+0000","last_peered":"2024-09-14T03:34:56.915927+0000","last_clean":"2024-09-14T03:34:56.915927+0000","last_became_active":"2024-09-14T03:34:02.519973+0000","last_became_peered":"2024-09-14T03:34:02.519973+0000","last_unstale":"2024-09-14T03:34:56.915927+0000","last_undegraded":"2024-09-14T03:34:56.915927+0000","last_fullsized":"2024-09-14T03:34:56.915927+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:20.217955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831096+0000","last_change":"2024-09-14T03:33:59.399709+0000","last_active":"2024-09-14T03:34:56.831096+0000","last_peered":"2024-09-14T03:34:56.831096+0000","last_clean":"2024-09-14T03:34:56.831096+0000","last_became_active":"2024-09-14T03:33:59.398533+0000","last_became_peered":"2024-09-14T03:33:59.398533+0000","last_unstale":"2024-09-14T03:34:56.831096+0000","last_undegraded":"2024-09-14T03:34:56.831096+0000","last_fullsized":"2024-09-14T03:34:56.831096+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:12:43.525175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917039+0000","last_change":"2024-09-14T03:34:02.520629+0000","last_active":"2024-09-14T03:34:56.917039+0000","last_peered":"2024-09-14T03:34:56.917039+0000","last_clean":"2024-09-14T03:34:56.917039+0000","last_became_active":"2024-09-14T03:34:02.520365+0000","last_became_peered":"2024-09-14T03:34:02.520365+0000","last_unstale":"2024-09-14T03:34:56.917039+0000","last_undegraded":"2024-09-14T03:34:56.917039+0000","last_fullsized":"2024-09-14T03:34:56.917039+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:29.519783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830800+0000","last_change":"2024-09-14T03:33:59.404858+0000","last_active":"2024-09-14T03:34:56.830800+0000","last_peered":"2024-09-14T03:34:56.830800+0000","last_clean":"2024-09-14T03:34:56.830800+0000","last_became_active":"2024-09-14T03:33:59.404261+0000","last_became_peered":"2024-09-14T03:33:59.404261+0000","last_unstale":"2024-09-14T03:34:56.830800+0000","last_undegraded":"2024-09-14T03:34:56.830800+0000","last_fullsized":"2024-09-14T03:34:56.830800+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:24:38.834576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916965+0000","last_change":"2024-09-14T03:34:02.516816+0000","last_active":"2024-09-14T03:34:56.916965+0000","last_peered":"2024-09-14T03:34:56.916965+0000","last_clean":"2024-09-14T03:34:56.916965+0000","last_became_active":"2024-09-14T03:34:02.516527+0000","last_became_peered":"2024-09-14T03:34:02.516527+0000","last_unstale":"2024-09-14T03:34:56.916965+0000","last_undegraded":"2024-09-14T03:34:56.916965+0000","last_fullsized":"2024-09-14T03:34:56.916965+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:35.627973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837514+0000","last_change":"2024-09-14T03:33:59.395532+0000","last_active":"2024-09-14T03:34:56.837514+0000","last_peered":"2024-09-14T03:34:56.837514+0000","last_clean":"2024-09-14T03:34:56.837514+0000","last_became_active":"2024-09-14T03:33:59.395255+0000","last_became_peered":"2024-09-14T03:33:59.395255+0000","last_unstale":"2024-09-14T03:34:56.837514+0000","last_undegraded":"2024-09-14T03:34:56.837514+0000","last_fullsized":"2024-09-14T03:34:56.837514+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:51.477913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839317+0000","last_change":"2024-09-14T03:34:02.520744+0000","last_active":"2024-09-14T03:34:56.839317+0000","last_peered":"2024-09-14T03:34:56.839317+0000","last_clean":"2024-09-14T03:34:56.839317+0000","last_became_active":"2024-09-14T03:34:02.520420+0000","last_became_peered":"2024-09-14T03:34:02.520420+0000","last_unstale":"2024-09-14T03:34:56.839317+0000","last_undegraded":"2024-09-14T03:34:56.839317+0000","last_fullsized":"2024-09-14T03:34:56.839317+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:32:43.573943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830896+0000","last_change":"2024-09-14T03:33:59.399299+0000","last_active":"2024-09-14T03:34:56.830896+0000","last_peered":"2024-09-14T03:34:56.830896+0000","last_clean":"2024-09-14T03:34:56.830896+0000","last_became_active":"2024-09-14T03:33:59.399008+0000","last_became_peered":"2024-09-14T03:33:59.399008+0000","last_unstale":"2024-09-14T03:34:56.830896+0000","last_undegraded":"2024-09-14T03:34:56.830896+0000","last_fullsized":"2024-09-14T03:34:56.830896+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:16:14.935721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916805+0000","last_change":"2024-09-14T03:34:02.521443+0000","last_active":"2024-09-14T03:34:56.916805+0000","last_peered":"2024-09-14T03:34:56.916805+0000","last_clean":"2024-09-14T03:34:56.916805+0000","last_became_active":"2024-09-14T03:34:02.519164+0000","last_became_peered":"2024-09-14T03:34:02.519164+0000","last_unstale":"2024-09-14T03:34:56.916805+0000","last_undegraded":"2024-09-14T03:34:56.916805+0000","last_fullsized":"2024-09-14T03:34:56.916805+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:28.199628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.293111+0000","last_change":"2024-09-14T03:34:56.917557+0000","last_active":"2024-09-14T03:35:05.293111+0000","last_peered":"2024-09-14T03:35:05.293111+0000","last_clean":"2024-09-14T03:35:05.293111+0000","last_became_active":"2024-09-14T03:34:52.000396+0000","last_became_peered":"2024-09-14T03:34:52.000396+0000","last_unstale":"2024-09-14T03:35:05.293111+0000","last_undegraded":"2024-09-14T03:35:05.293111+0000","last_fullsized":"2024-09-14T03:35:05.293111+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:18:00.378690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030274300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"62'79","reported_seq":58,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916460+0000","last_change":"2024-09-14T03:33:36.972974+0000","last_active":"2024-09-14T03:34:56.916460+0000","last_peered":"2024-09-14T03:34:56.916460+0000","last_clean":"2024-09-14T03:34:56.916460+0000","last_became_active":"2024-09-14T03:33:36.967695+0000","last_became_peered":"2024-09-14T03:33:36.967695+0000","last_unstale":"2024-09-14T03:34:56.916460+0000","last_undegraded":"2024-09-14T03:34:56.916460+0000","last_fullsized":"2024-09-14T03:34:56.916460+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_clean_scrub_stamp":"2024-09-14T03:31:39.719769+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:28:24.258573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839183+0000","last_change":"2024-09-14T03:33:59.406222+0000","last_active":"2024-09-14T03:34:56.839183+0000","last_peered":"2024-09-14T03:34:56.839183+0000","last_clean":"2024-09-14T03:34:56.839183+0000","last_became_active":"2024-09-14T03:33:59.405473+0000","last_became_peered":"2024-09-14T03:33:59.405473+0000","last_unstale":"2024-09-14T03:34:56.839183+0000","last_undegraded":"2024-09-14T03:34:56.839183+0000","last_fullsized":"2024-09-14T03:34:56.839183+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:26.326984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859862+0000","last_change":"2024-09-14T03:34:02.512416+0000","last_active":"2024-09-14T03:34:56.859862+0000","last_peered":"2024-09-14T03:34:56.859862+0000","last_clean":"2024-09-14T03:34:56.859862+0000","last_became_active":"2024-09-14T03:34:02.512135+0000","last_became_peered":"2024-09-14T03:34:02.512135+0000","last_unstale":"2024-09-14T03:34:56.859862+0000","last_undegraded":"2024-09-14T03:34:56.859862+0000","last_fullsized":"2024-09-14T03:34:56.859862+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:49.847882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:04.980477+0000","last_change":"2024-09-14T03:34:56.831437+0000","last_active":"2024-09-14T03:35:04.980477+0000","last_peered":"2024-09-14T03:35:04.980477+0000","last_clean":"2024-09-14T03:35:04.980477+0000","last_became_active":"2024-09-14T03:34:52.346558+0000","last_became_peered":"2024-09-14T03:34:52.346558+0000","last_unstale":"2024-09-14T03:35:04.980477+0000","last_undegraded":"2024-09-14T03:35:04.980477+0000","last_fullsized":"2024-09-14T03:35:04.980477+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:11.239970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050397699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916355+0000","last_change":"2024-09-14T03:33:59.409150+0000","last_active":"2024-09-14T03:34:56.916355+0000","last_peered":"2024-09-14T03:34:56.916355+0000","last_clean":"2024-09-14T03:34:56.916355+0000","last_became_active":"2024-09-14T03:33:59.408622+0000","last_became_peered":"2024-09-14T03:33:59.408622+0000","last_unstale":"2024-09-14T03:34:56.916355+0000","last_undegraded":"2024-09-14T03:34:56.916355+0000","last_fullsized":"2024-09-14T03:34:56.916355+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:11:03.508216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830380+0000","last_change":"2024-09-14T03:34:02.517068+0000","last_active":"2024-09-14T03:34:56.830380+0000","last_peered":"2024-09-14T03:34:56.830380+0000","last_clean":"2024-09-14T03:34:56.830380+0000","last_became_active":"2024-09-14T03:34:02.510158+0000","last_became_peered":"2024-09-14T03:34:02.510158+0000","last_unstale":"2024-09-14T03:34:56.830380+0000","last_undegraded":"2024-09-14T03:34:56.830380+0000","last_fullsized":"2024-09-14T03:34:56.830380+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:22.709122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.531485+0000","last_change":"2024-09-14T03:34:56.837898+0000","last_active":"2024-09-14T03:35:05.531485+0000","last_peered":"2024-09-14T03:35:05.531485+0000","last_clean":"2024-09-14T03:35:05.531485+0000","last_became_active":"2024-09-14T03:34:52.006563+0000","last_became_peered":"2024-09-14T03:34:52.006563+0000","last_unstale":"2024-09-14T03:35:05.531485+0000","last_undegraded":"2024-09-14T03:35:05.531485+0000","last_fullsized":"2024-09-14T03:35:05.531485+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:37.308715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027249499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830160+0000","last_change":"2024-09-14T03:33:59.399987+0000","last_active":"2024-09-14T03:34:56.830160+0000","last_peered":"2024-09-14T03:34:56.830160+0000","last_clean":"2024-09-14T03:34:56.830160+0000","last_became_active":"2024-09-14T03:33:59.399698+0000","last_became_peered":"2024-09-14T03:33:59.399698+0000","last_unstale":"2024-09-14T03:34:56.830160+0000","last_undegraded":"2024-09-14T03:34:56.830160+0000","last_fullsized":"2024-09-14T03:34:56.830160+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:58.806927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.836764+0000","last_change":"2024-09-14T03:34:02.515266+0000","last_active":"2024-09-14T03:34:56.836764+0000","last_peered":"2024-09-14T03:34:56.836764+0000","last_clean":"2024-09-14T03:34:56.836764+0000","last_became_active":"2024-09-14T03:34:02.515012+0000","last_became_peered":"2024-09-14T03:34:02.515012+0000","last_unstale":"2024-09-14T03:34:56.836764+0000","last_undegraded":"2024-09-14T03:34:56.836764+0000","last_fullsized":"2024-09-14T03:34:56.836764+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:52:23.510282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.531194+0000","last_change":"2024-09-14T03:34:56.837741+0000","last_active":"2024-09-14T03:35:05.531194+0000","last_peered":"2024-09-14T03:35:05.531194+0000","last_clean":"2024-09-14T03:35:05.531194+0000","last_became_active":"2024-09-14T03:34:52.346774+0000","last_became_peered":"2024-09-14T03:34:52.346774+0000","last_unstale":"2024-09-14T03:35:05.531194+0000","last_undegraded":"2024-09-14T03:35:05.531194+0000","last_fullsized":"2024-09-14T03:35:05.531194+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:53.687838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065452200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839454+0000","last_change":"2024-09-14T03:33:59.399736+0000","last_active":"2024-09-14T03:34:56.839454+0000","last_peered":"2024-09-14T03:34:56.839454+0000","last_clean":"2024-09-14T03:34:56.839454+0000","last_became_active":"2024-09-14T03:33:59.399434+0000","last_became_peered":"2024-09-14T03:33:59.399434+0000","last_unstale":"2024-09-14T03:34:56.839454+0000","last_undegraded":"2024-09-14T03:34:56.839454+0000","last_fullsized":"2024-09-14T03:34:56.839454+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:12.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830739+0000","last_change":"2024-09-14T03:34:02.507742+0000","last_active":"2024-09-14T03:34:56.830739+0000","last_peered":"2024-09-14T03:34:56.830739+0000","last_clean":"2024-09-14T03:34:56.830739+0000","last_became_active":"2024-09-14T03:34:02.507452+0000","last_became_peered":"2024-09-14T03:34:02.507452+0000","last_unstale":"2024-09-14T03:34:56.830739+0000","last_undegraded":"2024-09-14T03:34:56.830739+0000","last_fullsized":"2024-09-14T03:34:56.830739+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:50.407482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"67'2","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.530926+0000","last_change":"2024-09-14T03:34:56.837653+0000","last_active":"2024-09-14T03:35:05.530926+0000","last_peered":"2024-09-14T03:35:05.530926+0000","last_clean":"2024-09-14T03:35:05.530926+0000","last_became_active":"2024-09-14T03:34:52.005925+0000","last_became_peered":"2024-09-14T03:34:52.005925+0000","last_unstale":"2024-09-14T03:35:05.530926+0000","last_undegraded":"2024-09-14T03:35:05.530926+0000","last_fullsized":"2024-09-14T03:35:05.530926+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:57:41.107409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083349699999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837358+0000","last_change":"2024-09-14T03:33:59.400498+0000","last_active":"2024-09-14T03:34:56.837358+0000","last_peered":"2024-09-14T03:34:56.837358+0000","last_clean":"2024-09-14T03:34:56.837358+0000","last_became_active":"2024-09-14T03:33:59.400204+0000","last_became_peered":"2024-09-14T03:33:59.400204+0000","last_unstale":"2024-09-14T03:34:56.837358+0000","last_undegraded":"2024-09-14T03:34:56.837358+0000","last_fullsized":"2024-09-14T03:34:56.837358+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:07:48.735887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839417+0000","last_change":"2024-09-14T03:34:02.513933+0000","last_active":"2024-09-14T03:34:56.839417+0000","last_peered":"2024-09-14T03:34:56.839417+0000","last_clean":"2024-09-14T03:34:56.839417+0000","last_became_active":"2024-09-14T03:34:02.513689+0000","last_became_peered":"2024-09-14T03:34:02.513689+0000","last_unstale":"2024-09-14T03:34:56.839417+0000","last_undegraded":"2024-09-14T03:34:56.839417+0000","last_fullsized":"2024-09-14T03:34:56.839417+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:23.791135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"65'1","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:04.979871+0000","last_change":"2024-09-14T03:34:56.831590+0000","last_active":"2024-09-14T03:35:04.979871+0000","last_peered":"2024-09-14T03:35:04.979871+0000","last_clean":"2024-09-14T03:35:04.979871+0000","last_became_active":"2024-09-14T03:34:52.005945+0000","last_became_peered":"2024-09-14T03:34:52.005945+0000","last_unstale":"2024-09-14T03:35:04.979871+0000","last_undegraded":"2024-09-14T03:35:04.979871+0000","last_fullsized":"2024-09-14T03:35:04.979871+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:00.810273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058187600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830345+0000","last_change":"2024-09-14T03:34:34.954630+0000","last_active":"2024-09-14T03:34:56.830345+0000","last_peered":"2024-09-14T03:34:56.830345+0000","last_clean":"2024-09-14T03:34:56.830345+0000","last_became_active":"2024-09-14T03:34:34.954209+0000","last_became_peered":"2024-09-14T03:34:34.954209+0000","last_unstale":"2024-09-14T03:34:56.830345+0000","last_undegraded":"2024-09-14T03:34:56.830345+0000","last_fullsized":"2024-09-14T03:34:56.830345+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:57.878910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837283+0000","last_change":"2024-09-14T03:34:02.513000+0000","last_active":"2024-09-14T03:34:56.837283+0000","last_peered":"2024-09-14T03:34:56.837283+0000","last_clean":"2024-09-14T03:34:56.837283+0000","last_became_active":"2024-09-14T03:34:02.512679+0000","last_became_peered":"2024-09-14T03:34:02.512679+0000","last_unstale":"2024-09-14T03:34:56.837283+0000","last_undegraded":"2024-09-14T03:34:56.837283+0000","last_fullsized":"2024-09-14T03:34:56.837283+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:43:48.821069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.857050+0000","last_change":"2024-09-14T03:34:56.860310+0000","last_active":"2024-09-14T03:35:05.857050+0000","last_peered":"2024-09-14T03:35:05.857050+0000","last_clean":"2024-09-14T03:35:05.857050+0000","last_became_active":"2024-09-14T03:34:51.997457+0000","last_became_peered":"2024-09-14T03:34:51.997457+0000","last_unstale":"2024-09-14T03:35:05.857050+0000","last_undegraded":"2024-09-14T03:35:05.857050+0000","last_fullsized":"2024-09-14T03:35:05.857050+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:30.409340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070668399999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837791+0000","last_change":"2024-09-14T03:33:59.397629+0000","last_active":"2024-09-14T03:34:56.837791+0000","last_peered":"2024-09-14T03:34:56.837791+0000","last_clean":"2024-09-14T03:34:56.837791+0000","last_became_active":"2024-09-14T03:33:59.397377+0000","last_became_peered":"2024-09-14T03:33:59.397377+0000","last_unstale":"2024-09-14T03:34:56.837791+0000","last_undegraded":"2024-09-14T03:34:56.837791+0000","last_fullsized":"2024-09-14T03:34:56.837791+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:42:02.914679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.860019+0000","last_change":"2024-09-14T03:34:02.517598+0000","last_active":"2024-09-14T03:34:56.860019+0000","last_peered":"2024-09-14T03:34:56.860019+0000","last_clean":"2024-09-14T03:34:56.860019+0000","last_became_active":"2024-09-14T03:34:02.517367+0000","last_became_peered":"2024-09-14T03:34:02.517367+0000","last_unstale":"2024-09-14T03:34:56.860019+0000","last_undegraded":"2024-09-14T03:34:56.860019+0000","last_fullsized":"2024-09-14T03:34:56.860019+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:44.727080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.145236+0000","last_change":"2024-09-14T03:34:56.830903+0000","last_active":"2024-09-14T03:35:05.145236+0000","last_peered":"2024-09-14T03:35:05.145236+0000","last_clean":"2024-09-14T03:35:05.145236+0000","last_became_active":"2024-09-14T03:34:52.006158+0000","last_became_peered":"2024-09-14T03:34:52.006158+0000","last_unstale":"2024-09-14T03:35:05.145236+0000","last_undegraded":"2024-09-14T03:35:05.145236+0000","last_fullsized":"2024-09-14T03:35:05.145236+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:03.846292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059832600000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916770+0000","last_change":"2024-09-14T03:33:59.405301+0000","last_active":"2024-09-14T03:34:56.916770+0000","last_peered":"2024-09-14T03:34:56.916770+0000","last_clean":"2024-09-14T03:34:56.916770+0000","last_became_active":"2024-09-14T03:33:59.404994+0000","last_became_peered":"2024-09-14T03:33:59.404994+0000","last_unstale":"2024-09-14T03:34:56.916770+0000","last_undegraded":"2024-09-14T03:34:56.916770+0000","last_fullsized":"2024-09-14T03:34:56.916770+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:52:19.415253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830839+0000","last_change":"2024-09-14T03:34:02.516570+0000","last_active":"2024-09-14T03:34:56.830839+0000","last_peered":"2024-09-14T03:34:56.830839+0000","last_clean":"2024-09-14T03:34:56.830839+0000","last_became_active":"2024-09-14T03:34:02.516212+0000","last_became_peered":"2024-09-14T03:34:02.516212+0000","last_unstale":"2024-09-14T03:34:56.830839+0000","last_undegraded":"2024-09-14T03:34:56.830839+0000","last_fullsized":"2024-09-14T03:34:56.830839+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:33:23.282986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916200+0000","last_change":"2024-09-14T03:33:59.408805+0000","last_active":"2024-09-14T03:34:56.916200+0000","last_peered":"2024-09-14T03:34:56.916200+0000","last_clean":"2024-09-14T03:34:56.916200+0000","last_became_active":"2024-09-14T03:33:59.408521+0000","last_became_peered":"2024-09-14T03:33:59.408521+0000","last_unstale":"2024-09-14T03:34:56.916200+0000","last_undegraded":"2024-09-14T03:34:56.916200+0000","last_fullsized":"2024-09-14T03:34:56.916200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:20:00.847563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830664+0000","last_change":"2024-09-14T03:34:02.511411+0000","last_active":"2024-09-14T03:34:56.830664+0000","last_peered":"2024-09-14T03:34:56.830664+0000","last_clean":"2024-09-14T03:34:56.830664+0000","last_became_active":"2024-09-14T03:34:02.511131+0000","last_became_peered":"2024-09-14T03:34:02.511131+0000","last_unstale":"2024-09-14T03:34:56.830664+0000","last_undegraded":"2024-09-14T03:34:56.830664+0000","last_fullsized":"2024-09-14T03:34:56.830664+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:23.033860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.836906+0000","last_change":"2024-09-14T03:33:59.400483+0000","last_active":"2024-09-14T03:34:56.836906+0000","last_peered":"2024-09-14T03:34:56.836906+0000","last_clean":"2024-09-14T03:34:56.836906+0000","last_became_active":"2024-09-14T03:33:59.400172+0000","last_became_peered":"2024-09-14T03:33:59.400172+0000","last_unstale":"2024-09-14T03:34:56.836906+0000","last_undegraded":"2024-09-14T03:34:56.836906+0000","last_fullsized":"2024-09-14T03:34:56.836906+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:40.756478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859865+0000","last_change":"2024-09-14T03:34:02.505712+0000","last_active":"2024-09-14T03:34:56.859865+0000","last_peered":"2024-09-14T03:34:56.859865+0000","last_clean":"2024-09-14T03:34:56.859865+0000","last_became_active":"2024-09-14T03:34:02.505460+0000","last_became_peered":"2024-09-14T03:34:02.505460+0000","last_unstale":"2024-09-14T03:34:56.859865+0000","last_undegraded":"2024-09-14T03:34:56.859865+0000","last_fullsized":"2024-09-14T03:34:56.859865+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:20.752579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837257+0000","last_change":"2024-09-14T03:33:59.405640+0000","last_active":"2024-09-14T03:34:56.837257+0000","last_peered":"2024-09-14T03:34:56.837257+0000","last_clean":"2024-09-14T03:34:56.837257+0000","last_became_active":"2024-09-14T03:33:59.405382+0000","last_became_peered":"2024-09-14T03:33:59.405382+0000","last_unstale":"2024-09-14T03:34:56.837257+0000","last_undegraded":"2024-09-14T03:34:56.837257+0000","last_fullsized":"2024-09-14T03:34:56.837257+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:36.350128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859846+0000","last_change":"2024-09-14T03:34:02.516581+0000","last_active":"2024-09-14T03:34:56.859846+0000","last_peered":"2024-09-14T03:34:56.859846+0000","last_clean":"2024-09-14T03:34:56.859846+0000","last_became_active":"2024-09-14T03:34:02.516335+0000","last_became_peered":"2024-09-14T03:34:02.516335+0000","last_unstale":"2024-09-14T03:34:56.859846+0000","last_undegraded":"2024-09-14T03:34:56.859846+0000","last_fullsized":"2024-09-14T03:34:56.859846+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:06.070599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916927+0000","last_change":"2024-09-14T03:33:59.408573+0000","last_active":"2024-09-14T03:34:56.916927+0000","last_peered":"2024-09-14T03:34:56.916927+0000","last_clean":"2024-09-14T03:34:56.916927+0000","last_became_active":"2024-09-14T03:33:59.408264+0000","last_became_peered":"2024-09-14T03:33:59.408264+0000","last_unstale":"2024-09-14T03:34:56.916927+0000","last_undegraded":"2024-09-14T03:34:56.916927+0000","last_fullsized":"2024-09-14T03:34:56.916927+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:30.937449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916037+0000","last_change":"2024-09-14T03:34:02.516408+0000","last_active":"2024-09-14T03:34:56.916037+0000","last_peered":"2024-09-14T03:34:56.916037+0000","last_clean":"2024-09-14T03:34:56.916037+0000","last_became_active":"2024-09-14T03:34:02.516119+0000","last_became_peered":"2024-09-14T03:34:02.516119+0000","last_unstale":"2024-09-14T03:34:56.916037+0000","last_undegraded":"2024-09-14T03:34:56.916037+0000","last_fullsized":"2024-09-14T03:34:56.916037+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:38:16.570920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839391+0000","last_change":"2024-09-14T03:34:02.512557+0000","last_active":"2024-09-14T03:34:56.839391+0000","last_peered":"2024-09-14T03:34:56.839391+0000","last_clean":"2024-09-14T03:34:56.839391+0000","last_became_active":"2024-09-14T03:34:02.512183+0000","last_became_peered":"2024-09-14T03:34:02.512183+0000","last_unstale":"2024-09-14T03:34:56.839391+0000","last_undegraded":"2024-09-14T03:34:56.839391+0000","last_fullsized":"2024-09-14T03:34:56.839391+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:05:14.073579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916723+0000","last_change":"2024-09-14T03:33:59.406333+0000","last_active":"2024-09-14T03:34:56.916723+0000","last_peered":"2024-09-14T03:34:56.916723+0000","last_clean":"2024-09-14T03:34:56.916723+0000","last_became_active":"2024-09-14T03:33:59.406065+0000","last_became_peered":"2024-09-14T03:33:59.406065+0000","last_unstale":"2024-09-14T03:34:56.916723+0000","last_undegraded":"2024-09-14T03:34:56.916723+0000","last_fullsized":"2024-09-14T03:34:56.916723+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:04.009889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839294+0000","last_change":"2024-09-14T03:34:02.508363+0000","last_active":"2024-09-14T03:34:56.839294+0000","last_peered":"2024-09-14T03:34:56.839294+0000","last_clean":"2024-09-14T03:34:56.839294+0000","last_became_active":"2024-09-14T03:34:02.508086+0000","last_became_peered":"2024-09-14T03:34:02.508086+0000","last_unstale":"2024-09-14T03:34:56.839294+0000","last_undegraded":"2024-09-14T03:34:56.839294+0000","last_fullsized":"2024-09-14T03:34:56.839294+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:47.194036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916839+0000","last_change":"2024-09-14T03:33:59.406945+0000","last_active":"2024-09-14T03:34:56.916839+0000","last_peered":"2024-09-14T03:34:56.916839+0000","last_clean":"2024-09-14T03:34:56.916839+0000","last_became_active":"2024-09-14T03:33:59.406662+0000","last_became_peered":"2024-09-14T03:33:59.406662+0000","last_unstale":"2024-09-14T03:34:56.916839+0000","last_undegraded":"2024-09-14T03:34:56.916839+0000","last_fullsized":"2024-09-14T03:34:56.916839+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:46:42.749128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831327+0000","last_change":"2024-09-14T03:34:02.510922+0000","last_active":"2024-09-14T03:34:56.831327+0000","last_peered":"2024-09-14T03:34:56.831327+0000","last_clean":"2024-09-14T03:34:56.831327+0000","last_became_active":"2024-09-14T03:34:02.510663+0000","last_became_peered":"2024-09-14T03:34:02.510663+0000","last_unstale":"2024-09-14T03:34:56.831327+0000","last_undegraded":"2024-09-14T03:34:56.831327+0000","last_fullsized":"2024-09-14T03:34:56.831327+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:00:02.779787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.860081+0000","last_change":"2024-09-14T03:33:59.398992+0000","last_active":"2024-09-14T03:34:56.860081+0000","last_peered":"2024-09-14T03:34:56.860081+0000","last_clean":"2024-09-14T03:34:56.860081+0000","last_became_active":"2024-09-14T03:33:59.398725+0000","last_became_peered":"2024-09-14T03:33:59.398725+0000","last_unstale":"2024-09-14T03:34:56.860081+0000","last_undegraded":"2024-09-14T03:34:56.860081+0000","last_fullsized":"2024-09-14T03:34:56.860081+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:50:20.745864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859899+0000","last_change":"2024-09-14T03:34:02.514810+0000","last_active":"2024-09-14T03:34:56.859899+0000","last_peered":"2024-09-14T03:34:56.859899+0000","last_clean":"2024-09-14T03:34:56.859899+0000","last_became_active":"2024-09-14T03:34:02.514500+0000","last_became_peered":"2024-09-14T03:34:02.514500+0000","last_unstale":"2024-09-14T03:34:56.859899+0000","last_undegraded":"2024-09-14T03:34:56.859899+0000","last_fullsized":"2024-09-14T03:34:56.859899+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:20:21.110046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":28,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831482+0000","last_change":"2024-09-14T03:33:59.397898+0000","last_active":"2024-09-14T03:34:56.831482+0000","last_peered":"2024-09-14T03:34:56.831482+0000","last_clean":"2024-09-14T03:34:56.831482+0000","last_became_active":"2024-09-14T03:33:59.396670+0000","last_became_peered":"2024-09-14T03:33:59.396670+0000","last_unstale":"2024-09-14T03:34:56.831482+0000","last_undegraded":"2024-09-14T03:34:56.831482+0000","last_fullsized":"2024-09-14T03:34:56.831482+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:46.652243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397533,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":903104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.504,"15min":0.50800000000000001},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63900000000000001,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.52700000000000002,"15min":0.53500000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.64300000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.58299999999999996,"15min":0.59499999999999997},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67400000000000004,"5min":1.044,"15min":1.044},"last":0.437}]},{"osd":2,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.57699999999999996,"15min":0.59099999999999997},"min":{"1min":0.36099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.65100000000000002,"5min":1.012,"15min":1.012},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.54400000000000004,"15min":0.55600000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64100000000000001,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.46200000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.56399999999999995,"15min":0.57799999999999996},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.67300000000000004,"5min":1.115,"15min":1.115},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.57699999999999996,"15min":0.59099999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.69199999999999995,"5min":0.87,"15min":0.87},"last":0.51300000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.46999999999999997,"15min":0.47799999999999998},"min":{"1min":0.309,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60399999999999998,"5min":1.079,"15min":1.079},"last":0.504},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.51900000000000002,"15min":0.53100000000000003},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.70599999999999996,"5min":1.282,"15min":1.282},"last":0.32200000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.51500000000000001,"15min":0.53100000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68600000000000005,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.52200000000000002,"15min":0.52900000000000003},"min":{"1min":0.35599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.71799999999999997,"5min":1.135,"15min":1.135},"last":0.48099999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.58399999999999996,"15min":0.59799999999999998},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.75,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.58399999999999996,"15min":0.59999999999999998},"min":{"1min":0.37,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73599999999999999,"5min":1.625,"15min":1.625},"last":0.45200000000000001}]}]},{"osd":6,"up_from":43,"seq":184683593761,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":904539,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.374,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.52200000000000002,"15min":0.51900000000000002},"min":{"1min":0.42199999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.59299999999999997}]},{"osd":1,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51100000000000001,"15min":0.50900000000000001},"min":{"1min":0.38900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.65800000000000003}]},{"osd":3,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54900000000000004,"15min":0.55400000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72399999999999998,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.61699999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.442,"15min":0.44500000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.45900000000000002,"15min":0.46200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.64200000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.45600000000000002,"15min":0.45900000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.59899999999999998,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.48299999999999998,"15min":0.48699999999999999},"min":{"1min":0.32700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60299999999999998,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.47499999999999998,"15min":0.47099999999999997},"min":{"1min":0.33500000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.72999999999999998,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.51500000000000001,"15min":0.51100000000000001},"min":{"1min":0.34699999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.68600000000000005}]}]},{"osd":5,"up_from":37,"seq":158913789990,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":314036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.61699999999999999,"15min":0.55700000000000005},"min":{"1min":0.41899999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":11.026999999999999,"5min":11.026999999999999,"15min":11.026999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.52000000000000002,"15min":0.52200000000000002},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64400000000000002,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.42899999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.67300000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.51000000000000001,"15min":0.51400000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.64400000000000002,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.49399999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.51200000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.61899999999999999,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.505,"15min":0.50700000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.68200000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.41099999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52700000000000002,"15min":0.52800000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.55100000000000005}]},{"osd":4,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.28899999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.623},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.376,"15min":0.36399999999999999},"min":{"1min":0.32900000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.63400000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:35:17 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.46200000000000002,"15min":0.45800000000000002},"min":{"1min":0.30199999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.52000000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:35:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.505,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.48499999999999999,"15min":0.48199999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.68400000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.56599999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986219,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":312859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.499,"15min":0.50700000000000001},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63100000000000001,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.51200000000000001,"15min":0.51600000000000001},"min":{"1min":0.35799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.625,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.56200000000000006}]},{"osd":1,"last update":"Sat Sep 14 03:35:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.52600000000000002,"15min":0.53800000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68000000000000005,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.51300000000000001,"15min":0.51500000000000001},"min":{"1min":0.39100000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.68000000000000005,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.60699999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:35:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52900000000000003,"15min":0.53500000000000003},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.65700000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.52200000000000002,"15min":0.53600000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.63900000000000001,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.48199999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:35:29 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.54500000000000004,"15min":0.55700000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.63700000000000001,"5min":0.752,"15min":0.752},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.379,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.64300000000000002,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:34:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.36699999999999999,"15min":0.35099999999999998},"min":{"1min":0.31,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.437,"5min":0.36899999999999999,"15min":0.35799999999999998},"min":{"1min":0.30199999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.443}]},{"osd":6,"last update":"Sat Sep 14 03:35:18 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.40600000000000003,"15min":0.40000000000000002},"min":{"1min":0.32800000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.376},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.45000000000000001,"15min":0.443},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.42099999999999999}]},{"osd":7,"last update":"Sat Sep 14 03:35:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.71699999999999997,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.70799999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.505}]}]},{"osd":3,"up_from":26,"seq":111669149743,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":314590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:11 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.35499999999999998,"15min":0.35799999999999998},"min":{"1min":0.23100000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.41199999999999998,"15min":0.42399999999999999},"min":{"1min":0.30099999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 14 03:35:11 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.379,"15min":0.38700000000000001},"min":{"1min":0.27400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.51100000000000001,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.379,"5min":0.38500000000000001,"15min":0.39300000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:35:11 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.41499999999999998,"15min":0.42899999999999999},"min":{"1min":0.29399999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.31},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39400000000000002,"15min":0.40300000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.29799999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.56599999999999995,"15min":0.57299999999999995},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.379},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.432}]},{"osd":5,"last update":"Sat Sep 14 03:34:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56899999999999995,"15min":0.57299999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.438},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.69899999999999995,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.40600000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:35:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54200000000000004,"15min":0.53900000000000003},"min":{"1min":0.41199999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.55100000000000005,"15min":0.54600000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46100000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.41199999999999998,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.5}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":312144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:34:45 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.39700000000000002,"15min":0.40300000000000002},"min":{"1min":0.22900000000000001,"5min":0.16300000000000001,"15min":0.16300000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.42599999999999999,"15min":0.42899999999999999},"min":{"1min":0.29199999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.317}]},{"osd":1,"last update":"Sat Sep 14 03:34:45 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.377,"15min":0.379},"min":{"1min":0.23799999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.51000000000000001,"5min":0.621,"15min":0.621},"last":0.255},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.24399999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.47699999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:35:07 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.40100000000000002,"15min":0.41099999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.46100000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.245},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55900000000000005,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.30199999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.38500000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.71899999999999997,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.90800000000000003}]},{"osd":5,"last update":"Sat Sep 14 03:34:51 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56999999999999995,"15min":0.57099999999999995},"min":{"1min":0.45900000000000002,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.436,"5min":0.435,"15min":0.435},"max":{"1min":0.68500000000000005,"5min":0.745,"15min":0.745},"last":0.40699999999999997}]},{"osd":6,"last update":"Sat Sep 14 03:35:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.57099999999999995,"15min":0.58099999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.67000000000000004,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.58999999999999997,"15min":0.60499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.82299999999999995}]},{"osd":7,"last update":"Sat Sep 14 03:35:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54800000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.66200000000000003,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.436,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.78000000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574904,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":311378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:26 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34300000000000003,"15min":0.32000000000000001},"min":{"1min":0.24199999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.42499999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.32700000000000001,"5min":0.34399999999999997,"15min":0.33300000000000002},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.47399999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.371,"15min":0.39000000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.46000000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.39200000000000002,"15min":0.40600000000000003},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.40999999999999998,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.26600000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:35:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.437,"15min":0.45300000000000001},"min":{"1min":0.29199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.47199999999999998,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.442,"15min":0.46000000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.499,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:35:32 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.56999999999999995,"15min":0.57499999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60399999999999998,"15min":0.60999999999999999},"min":{"1min":0.42699999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.65000000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:34:55 2024","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":0.79000000000000004,"15min":0.66500000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":15.747999999999999,"5min":15.747999999999999,"15min":15.747999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.5649999999999999,"5min":0.77000000000000002,"15min":0.63800000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":15.760999999999999,"5min":15.760999999999999,"15min":15.760999999999999},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 14 03:35:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.446,"5min":0.89000000000000001,"15min":0.79800000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":15.775,"5min":15.775,"15min":15.775},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36899999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.86199999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.361,"15min":1.486},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.746,"5min":15.734,"15min":15.734},"last":0.622}]}]},{"osd":0,"up_from":8,"seq":34359738428,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298632,"kb_used_data":1276,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446520,"statfs":{"total":95995035648,"available":95689236480,"internally_reserved":0,"allocated":1306624,"data_stored":903228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:35:26 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.317},"min":{"1min":0.23999999999999999,"5min":0.192,"15min":0.192},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.33200000000000002},"min":{"1min":0.27000000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.56399999999999995}]},{"osd":2,"last update":"Sat Sep 14 03:35:43 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.371,"15min":0.35899999999999999},"min":{"1min":0.254,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.38300000000000001,"15min":0.374},"min":{"1min":0.27600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.54100000000000004}]},{"osd":3,"last update":"Sat Sep 14 03:35:08 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.42099999999999999,"15min":0.41999999999999998},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.502},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.44400000000000001,"15min":0.45400000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.48899999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.59899999999999998,"15min":0.60499999999999998},"min":{"1min":0.41899999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.1799999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.55600000000000005,"15min":0.55400000000000005},"min":{"1min":0.40300000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.72499999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:34:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54100000000000004,"15min":0.53800000000000003},"min":{"1min":0.39600000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:35:12 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.53800000000000003,"15min":0.52500000000000002},"min":{"1min":0.441,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.63900000000000001,"15min":0.63600000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.2210000000000001,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.53100000000000003}]},{"osd":7,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57899999999999996,"15min":0.57699999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.57099999999999995,"15min":0.56399999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.51700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:35:51.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:35:52.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:51 smithi110 ceph-mon[35374]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:35:52.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:51 smithi110 ceph-mon[27365]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:35:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:51 smithi195 ceph-mon[30695]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:35:52.341 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:53.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[35374]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:53.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:35:53.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:35:53.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[35374]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[35374]: Reconfiguring daemon alertmanager.a on smithi110 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[27365]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[27365]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:35:53.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:53 smithi110 ceph-mon[27365]: Reconfiguring daemon alertmanager.a on smithi110 2024-09-14T03:35:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:53 smithi195 ceph-mon[30695]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:53 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:53 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:53 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:35:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:53 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:35:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:53 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:35:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:53 smithi195 ceph-mon[30695]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:35:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:53 smithi195 ceph-mon[30695]: Reconfiguring daemon alertmanager.a on smithi110 2024-09-14T03:35:54.082 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:35:54.083 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2024-09-14T03:35:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:55 smithi110 ceph-mon[27365]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:55 smithi110 ceph-mon[27365]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:35:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:55 smithi110 ceph-mon[35374]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:55 smithi110 ceph-mon[35374]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:35:55.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:55 smithi195 ceph-mon[30695]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:55.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:55 smithi195 ceph-mon[30695]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:35:56.834 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-09-14T03:35:53.834790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386112,"kb_used_data":7264,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575104,"statfs":{"total":767960285184,"available":765516906496,"internally_reserved":0,"allocated":7438336,"data_stored":4275878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002534"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916845+0000","last_change":"2024-09-14T03:34:02.519367+0000","last_active":"2024-09-14T03:34:56.916845+0000","last_peered":"2024-09-14T03:34:56.916845+0000","last_clean":"2024-09-14T03:34:56.916845+0000","last_became_active":"2024-09-14T03:34:02.518530+0000","last_became_peered":"2024-09-14T03:34:02.518530+0000","last_unstale":"2024-09-14T03:34:56.916845+0000","last_undegraded":"2024-09-14T03:34:56.916845+0000","last_fullsized":"2024-09-14T03:34:56.916845+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:02:36.205002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837707+0000","last_change":"2024-09-14T03:34:02.515212+0000","last_active":"2024-09-14T03:34:56.837707+0000","last_peered":"2024-09-14T03:34:56.837707+0000","last_clean":"2024-09-14T03:34:56.837707+0000","last_became_active":"2024-09-14T03:34:02.514951+0000","last_became_peered":"2024-09-14T03:34:02.514951+0000","last_unstale":"2024-09-14T03:34:56.837707+0000","last_undegraded":"2024-09-14T03:34:56.837707+0000","last_fullsized":"2024-09-14T03:34:56.837707+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:30.742131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831163+0000","last_change":"2024-09-14T03:34:02.504777+0000","last_active":"2024-09-14T03:34:56.831163+0000","last_peered":"2024-09-14T03:34:56.831163+0000","last_clean":"2024-09-14T03:34:56.831163+0000","last_became_active":"2024-09-14T03:34:02.504550+0000","last_became_peered":"2024-09-14T03:34:02.504550+0000","last_unstale":"2024-09-14T03:34:56.831163+0000","last_undegraded":"2024-09-14T03:34:56.831163+0000","last_fullsized":"2024-09-14T03:34:56.831163+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:59.864492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839205+0000","last_change":"2024-09-14T03:34:02.521444+0000","last_active":"2024-09-14T03:34:56.839205+0000","last_peered":"2024-09-14T03:34:56.839205+0000","last_clean":"2024-09-14T03:34:56.839205+0000","last_became_active":"2024-09-14T03:34:02.521149+0000","last_became_peered":"2024-09-14T03:34:02.521149+0000","last_unstale":"2024-09-14T03:34:56.839205+0000","last_undegraded":"2024-09-14T03:34:56.839205+0000","last_fullsized":"2024-09-14T03:34:56.839205+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:10.004625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916561+0000","last_change":"2024-09-14T03:34:02.513133+0000","last_active":"2024-09-14T03:34:56.916561+0000","last_peered":"2024-09-14T03:34:56.916561+0000","last_clean":"2024-09-14T03:34:56.916561+0000","last_became_active":"2024-09-14T03:34:02.512580+0000","last_became_peered":"2024-09-14T03:34:02.512580+0000","last_unstale":"2024-09-14T03:34:56.916561+0000","last_undegraded":"2024-09-14T03:34:56.916561+0000","last_fullsized":"2024-09-14T03:34:56.916561+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:03.077106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839116+0000","last_change":"2024-09-14T03:34:02.508230+0000","last_active":"2024-09-14T03:34:56.839116+0000","last_peered":"2024-09-14T03:34:56.839116+0000","last_clean":"2024-09-14T03:34:56.839116+0000","last_became_active":"2024-09-14T03:34:02.507921+0000","last_became_peered":"2024-09-14T03:34:02.507921+0000","last_unstale":"2024-09-14T03:34:56.839116+0000","last_undegraded":"2024-09-14T03:34:56.839116+0000","last_fullsized":"2024-09-14T03:34:56.839116+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:37.418870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.915849+0000","last_change":"2024-09-14T03:34:02.515319+0000","last_active":"2024-09-14T03:34:56.915849+0000","last_peered":"2024-09-14T03:34:56.915849+0000","last_clean":"2024-09-14T03:34:56.915849+0000","last_became_active":"2024-09-14T03:34:02.515022+0000","last_became_peered":"2024-09-14T03:34:02.515022+0000","last_unstale":"2024-09-14T03:34:56.915849+0000","last_undegraded":"2024-09-14T03:34:56.915849+0000","last_fullsized":"2024-09-14T03:34:56.915849+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:51:51.089278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859702+0000","last_change":"2024-09-14T03:34:02.522349+0000","last_active":"2024-09-14T03:34:56.859702+0000","last_peered":"2024-09-14T03:34:56.859702+0000","last_clean":"2024-09-14T03:34:56.859702+0000","last_became_active":"2024-09-14T03:34:02.522063+0000","last_became_peered":"2024-09-14T03:34:02.522063+0000","last_unstale":"2024-09-14T03:34:56.859702+0000","last_undegraded":"2024-09-14T03:34:56.859702+0000","last_fullsized":"2024-09-14T03:34:56.859702+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:15.378220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917135+0000","last_change":"2024-09-14T03:34:02.512031+0000","last_active":"2024-09-14T03:34:56.917135+0000","last_peered":"2024-09-14T03:34:56.917135+0000","last_clean":"2024-09-14T03:34:56.917135+0000","last_became_active":"2024-09-14T03:34:02.511364+0000","last_became_peered":"2024-09-14T03:34:02.511364+0000","last_unstale":"2024-09-14T03:34:56.917135+0000","last_undegraded":"2024-09-14T03:34:56.917135+0000","last_fullsized":"2024-09-14T03:34:56.917135+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:54:15.711934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917254+0000","last_change":"2024-09-14T03:34:02.518932+0000","last_active":"2024-09-14T03:34:56.917254+0000","last_peered":"2024-09-14T03:34:56.917254+0000","last_clean":"2024-09-14T03:34:56.917254+0000","last_became_active":"2024-09-14T03:34:02.518575+0000","last_became_peered":"2024-09-14T03:34:02.518575+0000","last_unstale":"2024-09-14T03:34:56.917254+0000","last_undegraded":"2024-09-14T03:34:56.917254+0000","last_fullsized":"2024-09-14T03:34:56.917254+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:01:11.929183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917339+0000","last_change":"2024-09-14T03:34:02.517977+0000","last_active":"2024-09-14T03:34:56.917339+0000","last_peered":"2024-09-14T03:34:56.917339+0000","last_clean":"2024-09-14T03:34:56.917339+0000","last_became_active":"2024-09-14T03:34:02.517385+0000","last_became_peered":"2024-09-14T03:34:02.517385+0000","last_unstale":"2024-09-14T03:34:56.917339+0000","last_undegraded":"2024-09-14T03:34:56.917339+0000","last_fullsized":"2024-09-14T03:34:56.917339+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:50.900533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830573+0000","last_change":"2024-09-14T03:34:02.510949+0000","last_active":"2024-09-14T03:34:56.830573+0000","last_peered":"2024-09-14T03:34:56.830573+0000","last_clean":"2024-09-14T03:34:56.830573+0000","last_became_active":"2024-09-14T03:34:02.510519+0000","last_became_peered":"2024-09-14T03:34:02.510519+0000","last_unstale":"2024-09-14T03:34:56.830573+0000","last_undegraded":"2024-09-14T03:34:56.830573+0000","last_fullsized":"2024-09-14T03:34:56.830573+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:06:16.860908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859767+0000","last_change":"2024-09-14T03:34:02.512463+0000","last_active":"2024-09-14T03:34:56.859767+0000","last_peered":"2024-09-14T03:34:56.859767+0000","last_clean":"2024-09-14T03:34:56.859767+0000","last_became_active":"2024-09-14T03:34:02.512201+0000","last_became_peered":"2024-09-14T03:34:02.512201+0000","last_unstale":"2024-09-14T03:34:56.859767+0000","last_undegraded":"2024-09-14T03:34:56.859767+0000","last_fullsized":"2024-09-14T03:34:56.859767+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:46.716482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830640+0000","last_change":"2024-09-14T03:34:02.513215+0000","last_active":"2024-09-14T03:34:56.830640+0000","last_peered":"2024-09-14T03:34:56.830640+0000","last_clean":"2024-09-14T03:34:56.830640+0000","last_became_active":"2024-09-14T03:34:02.512773+0000","last_became_peered":"2024-09-14T03:34:02.512773+0000","last_unstale":"2024-09-14T03:34:56.830640+0000","last_undegraded":"2024-09-14T03:34:56.830640+0000","last_fullsized":"2024-09-14T03:34:56.830640+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:27:04.454480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.836868+0000","last_change":"2024-09-14T03:34:02.503785+0000","last_active":"2024-09-14T03:34:56.836868+0000","last_peered":"2024-09-14T03:34:56.836868+0000","last_clean":"2024-09-14T03:34:56.836868+0000","last_became_active":"2024-09-14T03:34:02.503212+0000","last_became_peered":"2024-09-14T03:34:02.503212+0000","last_unstale":"2024-09-14T03:34:56.836868+0000","last_undegraded":"2024-09-14T03:34:56.836868+0000","last_fullsized":"2024-09-14T03:34:56.836868+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:15.408242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":31,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.838932+0000","last_change":"2024-09-14T03:34:34.953866+0000","last_active":"2024-09-14T03:34:56.838932+0000","last_peered":"2024-09-14T03:34:56.838932+0000","last_clean":"2024-09-14T03:34:56.838932+0000","last_became_active":"2024-09-14T03:34:34.953477+0000","last_became_peered":"2024-09-14T03:34:34.953477+0000","last_unstale":"2024-09-14T03:34:56.838932+0000","last_undegraded":"2024-09-14T03:34:56.838932+0000","last_fullsized":"2024-09-14T03:34:56.838932+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:22.262447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839661+0000","last_change":"2024-09-14T03:34:02.507186+0000","last_active":"2024-09-14T03:34:56.839661+0000","last_peered":"2024-09-14T03:34:56.839661+0000","last_clean":"2024-09-14T03:34:56.839661+0000","last_became_active":"2024-09-14T03:34:02.506907+0000","last_became_peered":"2024-09-14T03:34:02.506907+0000","last_unstale":"2024-09-14T03:34:56.839661+0000","last_undegraded":"2024-09-14T03:34:56.839661+0000","last_fullsized":"2024-09-14T03:34:56.839661+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:10:47.570717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830684+0000","last_change":"2024-09-14T03:34:02.519104+0000","last_active":"2024-09-14T03:34:56.830684+0000","last_peered":"2024-09-14T03:34:56.830684+0000","last_clean":"2024-09-14T03:34:56.830684+0000","last_became_active":"2024-09-14T03:34:02.518795+0000","last_became_peered":"2024-09-14T03:34:02.518795+0000","last_unstale":"2024-09-14T03:34:56.830684+0000","last_undegraded":"2024-09-14T03:34:56.830684+0000","last_fullsized":"2024-09-14T03:34:56.830684+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:34:49.356603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916926+0000","last_change":"2024-09-14T03:34:02.521848+0000","last_active":"2024-09-14T03:34:56.916926+0000","last_peered":"2024-09-14T03:34:56.916926+0000","last_clean":"2024-09-14T03:34:56.916926+0000","last_became_active":"2024-09-14T03:34:02.521559+0000","last_became_peered":"2024-09-14T03:34:02.521559+0000","last_unstale":"2024-09-14T03:34:56.916926+0000","last_undegraded":"2024-09-14T03:34:56.916926+0000","last_fullsized":"2024-09-14T03:34:56.916926+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:21:09.742073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916978+0000","last_change":"2024-09-14T03:34:02.515094+0000","last_active":"2024-09-14T03:34:56.916978+0000","last_peered":"2024-09-14T03:34:56.916978+0000","last_clean":"2024-09-14T03:34:56.916978+0000","last_became_active":"2024-09-14T03:34:02.514291+0000","last_became_peered":"2024-09-14T03:34:02.514291+0000","last_unstale":"2024-09-14T03:34:56.916978+0000","last_undegraded":"2024-09-14T03:34:56.916978+0000","last_fullsized":"2024-09-14T03:34:56.916978+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:39:52.842240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831065+0000","last_change":"2024-09-14T03:34:02.510758+0000","last_active":"2024-09-14T03:34:56.831065+0000","last_peered":"2024-09-14T03:34:56.831065+0000","last_clean":"2024-09-14T03:34:56.831065+0000","last_became_active":"2024-09-14T03:34:02.510485+0000","last_became_peered":"2024-09-14T03:34:02.510485+0000","last_unstale":"2024-09-14T03:34:56.831065+0000","last_undegraded":"2024-09-14T03:34:56.831065+0000","last_fullsized":"2024-09-14T03:34:56.831065+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:26:25.102759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839132+0000","last_change":"2024-09-14T03:34:02.511696+0000","last_active":"2024-09-14T03:34:56.839132+0000","last_peered":"2024-09-14T03:34:56.839132+0000","last_clean":"2024-09-14T03:34:56.839132+0000","last_became_active":"2024-09-14T03:34:02.511289+0000","last_became_peered":"2024-09-14T03:34:02.511289+0000","last_unstale":"2024-09-14T03:34:56.839132+0000","last_undegraded":"2024-09-14T03:34:56.839132+0000","last_fullsized":"2024-09-14T03:34:56.839132+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:02:27.657439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859569+0000","last_change":"2024-09-14T03:34:02.517564+0000","last_active":"2024-09-14T03:34:56.859569+0000","last_peered":"2024-09-14T03:34:56.859569+0000","last_clean":"2024-09-14T03:34:56.859569+0000","last_became_active":"2024-09-14T03:34:02.517314+0000","last_became_peered":"2024-09-14T03:34:02.517314+0000","last_unstale":"2024-09-14T03:34:56.859569+0000","last_undegraded":"2024-09-14T03:34:56.859569+0000","last_fullsized":"2024-09-14T03:34:56.859569+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:08.360046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.838946+0000","last_change":"2024-09-14T03:34:02.513865+0000","last_active":"2024-09-14T03:34:56.838946+0000","last_peered":"2024-09-14T03:34:56.838946+0000","last_clean":"2024-09-14T03:34:56.838946+0000","last_became_active":"2024-09-14T03:34:02.513008+0000","last_became_peered":"2024-09-14T03:34:02.513008+0000","last_unstale":"2024-09-14T03:34:56.838946+0000","last_undegraded":"2024-09-14T03:34:56.838946+0000","last_fullsized":"2024-09-14T03:34:56.838946+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:18:37.581596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859955+0000","last_change":"2024-09-14T03:34:02.517481+0000","last_active":"2024-09-14T03:34:56.859955+0000","last_peered":"2024-09-14T03:34:56.859955+0000","last_clean":"2024-09-14T03:34:56.859955+0000","last_became_active":"2024-09-14T03:34:02.517252+0000","last_became_peered":"2024-09-14T03:34:02.517252+0000","last_unstale":"2024-09-14T03:34:56.859955+0000","last_undegraded":"2024-09-14T03:34:56.859955+0000","last_fullsized":"2024-09-14T03:34:56.859955+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:29:15.209238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917253+0000","last_change":"2024-09-14T03:34:02.521201+0000","last_active":"2024-09-14T03:34:56.917253+0000","last_peered":"2024-09-14T03:34:56.917253+0000","last_clean":"2024-09-14T03:34:56.917253+0000","last_became_active":"2024-09-14T03:34:02.520764+0000","last_became_peered":"2024-09-14T03:34:02.520764+0000","last_unstale":"2024-09-14T03:34:56.917253+0000","last_undegraded":"2024-09-14T03:34:56.917253+0000","last_fullsized":"2024-09-14T03:34:56.917253+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:39:40.688146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837307+0000","last_change":"2024-09-14T03:34:02.504156+0000","last_active":"2024-09-14T03:34:56.837307+0000","last_peered":"2024-09-14T03:34:56.837307+0000","last_clean":"2024-09-14T03:34:56.837307+0000","last_became_active":"2024-09-14T03:34:02.503906+0000","last_became_peered":"2024-09-14T03:34:02.503906+0000","last_unstale":"2024-09-14T03:34:56.837307+0000","last_undegraded":"2024-09-14T03:34:56.837307+0000","last_fullsized":"2024-09-14T03:34:56.837307+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:33:57.350618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":31,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859822+0000","last_change":"2024-09-14T03:34:34.330855+0000","last_active":"2024-09-14T03:34:56.859822+0000","last_peered":"2024-09-14T03:34:56.859822+0000","last_clean":"2024-09-14T03:34:56.859822+0000","last_became_active":"2024-09-14T03:34:34.330493+0000","last_became_peered":"2024-09-14T03:34:34.330493+0000","last_unstale":"2024-09-14T03:34:56.859822+0000","last_undegraded":"2024-09-14T03:34:56.859822+0000","last_fullsized":"2024-09-14T03:34:56.859822+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:06.278074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830387+0000","last_change":"2024-09-14T03:34:34.955446+0000","last_active":"2024-09-14T03:34:56.830387+0000","last_peered":"2024-09-14T03:34:56.830387+0000","last_clean":"2024-09-14T03:34:56.830387+0000","last_became_active":"2024-09-14T03:34:34.955095+0000","last_became_peered":"2024-09-14T03:34:34.955095+0000","last_unstale":"2024-09-14T03:34:56.830387+0000","last_undegraded":"2024-09-14T03:34:56.830387+0000","last_fullsized":"2024-09-14T03:34:56.830387+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:01.417881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839234+0000","last_change":"2024-09-14T03:34:02.513294+0000","last_active":"2024-09-14T03:34:56.839234+0000","last_peered":"2024-09-14T03:34:56.839234+0000","last_clean":"2024-09-14T03:34:56.839234+0000","last_became_active":"2024-09-14T03:34:02.512998+0000","last_became_peered":"2024-09-14T03:34:02.512998+0000","last_unstale":"2024-09-14T03:34:56.839234+0000","last_undegraded":"2024-09-14T03:34:56.839234+0000","last_fullsized":"2024-09-14T03:34:56.839234+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:54.262830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830045+0000","last_change":"2024-09-14T03:34:02.512389+0000","last_active":"2024-09-14T03:34:56.830045+0000","last_peered":"2024-09-14T03:34:56.830045+0000","last_clean":"2024-09-14T03:34:56.830045+0000","last_became_active":"2024-09-14T03:34:02.511955+0000","last_became_peered":"2024-09-14T03:34:02.511955+0000","last_unstale":"2024-09-14T03:34:56.830045+0000","last_undegraded":"2024-09-14T03:34:56.830045+0000","last_fullsized":"2024-09-14T03:34:56.830045+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:30:31.454464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.915824+0000","last_change":"2024-09-14T03:34:02.520047+0000","last_active":"2024-09-14T03:34:56.915824+0000","last_peered":"2024-09-14T03:34:56.915824+0000","last_clean":"2024-09-14T03:34:56.915824+0000","last_became_active":"2024-09-14T03:34:02.519764+0000","last_became_peered":"2024-09-14T03:34:02.519764+0000","last_unstale":"2024-09-14T03:34:56.915824+0000","last_undegraded":"2024-09-14T03:34:56.915824+0000","last_fullsized":"2024-09-14T03:34:56.915824+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:47:55.233617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830465+0000","last_change":"2024-09-14T03:34:02.515661+0000","last_active":"2024-09-14T03:34:56.830465+0000","last_peered":"2024-09-14T03:34:56.830465+0000","last_clean":"2024-09-14T03:34:56.830465+0000","last_became_active":"2024-09-14T03:34:02.515374+0000","last_became_peered":"2024-09-14T03:34:02.515374+0000","last_unstale":"2024-09-14T03:34:56.830465+0000","last_undegraded":"2024-09-14T03:34:56.830465+0000","last_fullsized":"2024-09-14T03:34:56.830465+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:06.066379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859940+0000","last_change":"2024-09-14T03:33:59.400168+0000","last_active":"2024-09-14T03:34:56.859940+0000","last_peered":"2024-09-14T03:34:56.859940+0000","last_clean":"2024-09-14T03:34:56.859940+0000","last_became_active":"2024-09-14T03:33:59.399904+0000","last_became_peered":"2024-09-14T03:33:59.399904+0000","last_unstale":"2024-09-14T03:34:56.859940+0000","last_undegraded":"2024-09-14T03:34:56.859940+0000","last_fullsized":"2024-09-14T03:34:56.859940+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.387300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831124+0000","last_change":"2024-09-14T03:34:02.514049+0000","last_active":"2024-09-14T03:34:56.831124+0000","last_peered":"2024-09-14T03:34:56.831124+0000","last_clean":"2024-09-14T03:34:56.831124+0000","last_became_active":"2024-09-14T03:34:02.513309+0000","last_became_peered":"2024-09-14T03:34:02.513309+0000","last_unstale":"2024-09-14T03:34:56.831124+0000","last_undegraded":"2024-09-14T03:34:56.831124+0000","last_fullsized":"2024-09-14T03:34:56.831124+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:14.127299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916208+0000","last_change":"2024-09-14T03:33:59.406006+0000","last_active":"2024-09-14T03:34:56.916208+0000","last_peered":"2024-09-14T03:34:56.916208+0000","last_clean":"2024-09-14T03:34:56.916208+0000","last_became_active":"2024-09-14T03:33:59.405514+0000","last_became_peered":"2024-09-14T03:33:59.405514+0000","last_unstale":"2024-09-14T03:34:56.916208+0000","last_undegraded":"2024-09-14T03:34:56.916208+0000","last_fullsized":"2024-09-14T03:34:56.916208+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:23:17.405355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859715+0000","last_change":"2024-09-14T03:33:59.399102+0000","last_active":"2024-09-14T03:34:56.859715+0000","last_peered":"2024-09-14T03:34:56.859715+0000","last_clean":"2024-09-14T03:34:56.859715+0000","last_became_active":"2024-09-14T03:33:59.398766+0000","last_became_peered":"2024-09-14T03:33:59.398766+0000","last_unstale":"2024-09-14T03:34:56.859715+0000","last_undegraded":"2024-09-14T03:34:56.859715+0000","last_fullsized":"2024-09-14T03:34:56.859715+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:08:01.084059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859784+0000","last_change":"2024-09-14T03:34:02.520106+0000","last_active":"2024-09-14T03:34:56.859784+0000","last_peered":"2024-09-14T03:34:56.859784+0000","last_clean":"2024-09-14T03:34:56.859784+0000","last_became_active":"2024-09-14T03:34:02.519825+0000","last_became_peered":"2024-09-14T03:34:02.519825+0000","last_unstale":"2024-09-14T03:34:56.859784+0000","last_undegraded":"2024-09-14T03:34:56.859784+0000","last_fullsized":"2024-09-14T03:34:56.859784+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:44.762587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839014+0000","last_change":"2024-09-14T03:33:59.406587+0000","last_active":"2024-09-14T03:34:56.839014+0000","last_peered":"2024-09-14T03:34:56.839014+0000","last_clean":"2024-09-14T03:34:56.839014+0000","last_became_active":"2024-09-14T03:33:59.405828+0000","last_became_peered":"2024-09-14T03:33:59.405828+0000","last_unstale":"2024-09-14T03:34:56.839014+0000","last_undegraded":"2024-09-14T03:34:56.839014+0000","last_fullsized":"2024-09-14T03:34:56.839014+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:57.857900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.836777+0000","last_change":"2024-09-14T03:34:02.507004+0000","last_active":"2024-09-14T03:34:56.836777+0000","last_peered":"2024-09-14T03:34:56.836777+0000","last_clean":"2024-09-14T03:34:56.836777+0000","last_became_active":"2024-09-14T03:34:02.506311+0000","last_became_peered":"2024-09-14T03:34:02.506311+0000","last_unstale":"2024-09-14T03:34:56.836777+0000","last_undegraded":"2024-09-14T03:34:56.836777+0000","last_fullsized":"2024-09-14T03:34:56.836777+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:33.038366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916191+0000","last_change":"2024-09-14T03:33:59.402116+0000","last_active":"2024-09-14T03:34:56.916191+0000","last_peered":"2024-09-14T03:34:56.916191+0000","last_clean":"2024-09-14T03:34:56.916191+0000","last_became_active":"2024-09-14T03:33:59.400274+0000","last_became_peered":"2024-09-14T03:33:59.400274+0000","last_unstale":"2024-09-14T03:34:56.916191+0000","last_undegraded":"2024-09-14T03:34:56.916191+0000","last_fullsized":"2024-09-14T03:34:56.916191+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:46.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839481+0000","last_change":"2024-09-14T03:34:02.520187+0000","last_active":"2024-09-14T03:34:56.839481+0000","last_peered":"2024-09-14T03:34:56.839481+0000","last_clean":"2024-09-14T03:34:56.839481+0000","last_became_active":"2024-09-14T03:34:02.519909+0000","last_became_peered":"2024-09-14T03:34:02.519909+0000","last_unstale":"2024-09-14T03:34:56.839481+0000","last_undegraded":"2024-09-14T03:34:56.839481+0000","last_fullsized":"2024-09-14T03:34:56.839481+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:32.588302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839588+0000","last_change":"2024-09-14T03:33:59.402069+0000","last_active":"2024-09-14T03:34:56.839588+0000","last_peered":"2024-09-14T03:34:56.839588+0000","last_clean":"2024-09-14T03:34:56.839588+0000","last_became_active":"2024-09-14T03:33:59.401698+0000","last_became_peered":"2024-09-14T03:33:59.401698+0000","last_unstale":"2024-09-14T03:34:56.839588+0000","last_undegraded":"2024-09-14T03:34:56.839588+0000","last_fullsized":"2024-09-14T03:34:56.839588+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:16:48.483610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831335+0000","last_change":"2024-09-14T03:34:02.515495+0000","last_active":"2024-09-14T03:34:56.831335+0000","last_peered":"2024-09-14T03:34:56.831335+0000","last_clean":"2024-09-14T03:34:56.831335+0000","last_became_active":"2024-09-14T03:34:02.515256+0000","last_became_peered":"2024-09-14T03:34:02.515256+0000","last_unstale":"2024-09-14T03:34:56.831335+0000","last_undegraded":"2024-09-14T03:34:56.831335+0000","last_fullsized":"2024-09-14T03:34:56.831335+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:02.392960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":27,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837428+0000","last_change":"2024-09-14T03:33:59.398566+0000","last_active":"2024-09-14T03:34:56.837428+0000","last_peered":"2024-09-14T03:34:56.837428+0000","last_clean":"2024-09-14T03:34:56.837428+0000","last_became_active":"2024-09-14T03:33:59.398281+0000","last_became_peered":"2024-09-14T03:33:59.398281+0000","last_unstale":"2024-09-14T03:34:56.837428+0000","last_undegraded":"2024-09-14T03:34:56.837428+0000","last_fullsized":"2024-09-14T03:34:56.837428+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:42.503990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830716+0000","last_change":"2024-09-14T03:34:34.953658+0000","last_active":"2024-09-14T03:34:56.830716+0000","last_peered":"2024-09-14T03:34:56.830716+0000","last_clean":"2024-09-14T03:34:56.830716+0000","last_became_active":"2024-09-14T03:34:34.953335+0000","last_became_peered":"2024-09-14T03:34:34.953335+0000","last_unstale":"2024-09-14T03:34:56.830716+0000","last_undegraded":"2024-09-14T03:34:56.830716+0000","last_fullsized":"2024-09-14T03:34:56.830716+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:40.796694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.14","version":"61'9","reported_seq":33,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917005+0000","last_change":"2024-09-14T03:33:59.401007+0000","last_active":"2024-09-14T03:34:56.917005+0000","last_peered":"2024-09-14T03:34:56.917005+0000","last_clean":"2024-09-14T03:34:56.917005+0000","last_became_active":"2024-09-14T03:33:59.400453+0000","last_became_peered":"2024-09-14T03:33:59.400453+0000","last_unstale":"2024-09-14T03:34:56.917005+0000","last_undegraded":"2024-09-14T03:34:56.917005+0000","last_fullsized":"2024-09-14T03:34:56.917005+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:32.633267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916702+0000","last_change":"2024-09-14T03:34:02.511714+0000","last_active":"2024-09-14T03:34:56.916702+0000","last_peered":"2024-09-14T03:34:56.916702+0000","last_clean":"2024-09-14T03:34:56.916702+0000","last_became_active":"2024-09-14T03:34:02.511328+0000","last_became_peered":"2024-09-14T03:34:02.511328+0000","last_unstale":"2024-09-14T03:34:56.916702+0000","last_undegraded":"2024-09-14T03:34:56.916702+0000","last_fullsized":"2024-09-14T03:34:56.916702+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:00:06.445335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831162+0000","last_change":"2024-09-14T03:33:59.396984+0000","last_active":"2024-09-14T03:34:56.831162+0000","last_peered":"2024-09-14T03:34:56.831162+0000","last_clean":"2024-09-14T03:34:56.831162+0000","last_became_active":"2024-09-14T03:33:59.395835+0000","last_became_peered":"2024-09-14T03:33:59.395835+0000","last_unstale":"2024-09-14T03:34:56.831162+0000","last_undegraded":"2024-09-14T03:34:56.831162+0000","last_fullsized":"2024-09-14T03:34:56.831162+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:57.889158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830709+0000","last_change":"2024-09-14T03:34:02.521192+0000","last_active":"2024-09-14T03:34:56.830709+0000","last_peered":"2024-09-14T03:34:56.830709+0000","last_clean":"2024-09-14T03:34:56.830709+0000","last_became_active":"2024-09-14T03:34:02.520704+0000","last_became_peered":"2024-09-14T03:34:02.520704+0000","last_unstale":"2024-09-14T03:34:56.830709+0000","last_undegraded":"2024-09-14T03:34:56.830709+0000","last_fullsized":"2024-09-14T03:34:56.830709+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:37:13.469199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839263+0000","last_change":"2024-09-14T03:33:59.406737+0000","last_active":"2024-09-14T03:34:56.839263+0000","last_peered":"2024-09-14T03:34:56.839263+0000","last_clean":"2024-09-14T03:34:56.839263+0000","last_became_active":"2024-09-14T03:33:59.405882+0000","last_became_peered":"2024-09-14T03:33:59.405882+0000","last_unstale":"2024-09-14T03:34:56.839263+0000","last_undegraded":"2024-09-14T03:34:56.839263+0000","last_fullsized":"2024-09-14T03:34:56.839263+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:58.379549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916562+0000","last_change":"2024-09-14T03:34:02.517299+0000","last_active":"2024-09-14T03:34:56.916562+0000","last_peered":"2024-09-14T03:34:56.916562+0000","last_clean":"2024-09-14T03:34:56.916562+0000","last_became_active":"2024-09-14T03:34:02.516007+0000","last_became_peered":"2024-09-14T03:34:02.516007+0000","last_unstale":"2024-09-14T03:34:56.916562+0000","last_undegraded":"2024-09-14T03:34:56.916562+0000","last_fullsized":"2024-09-14T03:34:56.916562+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:49.173576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916024+0000","last_change":"2024-09-14T03:33:59.401544+0000","last_active":"2024-09-14T03:34:56.916024+0000","last_peered":"2024-09-14T03:34:56.916024+0000","last_clean":"2024-09-14T03:34:56.916024+0000","last_became_active":"2024-09-14T03:33:59.400073+0000","last_became_peered":"2024-09-14T03:33:59.400073+0000","last_unstale":"2024-09-14T03:34:56.916024+0000","last_undegraded":"2024-09-14T03:34:56.916024+0000","last_fullsized":"2024-09-14T03:34:56.916024+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:24.935055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916207+0000","last_change":"2024-09-14T03:34:02.516939+0000","last_active":"2024-09-14T03:34:56.916207+0000","last_peered":"2024-09-14T03:34:56.916207+0000","last_clean":"2024-09-14T03:34:56.916207+0000","last_became_active":"2024-09-14T03:34:02.516657+0000","last_became_peered":"2024-09-14T03:34:02.516657+0000","last_unstale":"2024-09-14T03:34:56.916207+0000","last_undegraded":"2024-09-14T03:34:56.916207+0000","last_fullsized":"2024-09-14T03:34:56.916207+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:17.423269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":32,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830004+0000","last_change":"2024-09-14T03:33:59.400570+0000","last_active":"2024-09-14T03:34:56.830004+0000","last_peered":"2024-09-14T03:34:56.830004+0000","last_clean":"2024-09-14T03:34:56.830004+0000","last_became_active":"2024-09-14T03:33:59.400323+0000","last_became_peered":"2024-09-14T03:33:59.400323+0000","last_unstale":"2024-09-14T03:34:56.830004+0000","last_undegraded":"2024-09-14T03:34:56.830004+0000","last_fullsized":"2024-09-14T03:34:56.830004+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:39:46.484991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.915927+0000","last_change":"2024-09-14T03:34:02.521031+0000","last_active":"2024-09-14T03:34:56.915927+0000","last_peered":"2024-09-14T03:34:56.915927+0000","last_clean":"2024-09-14T03:34:56.915927+0000","last_became_active":"2024-09-14T03:34:02.519973+0000","last_became_peered":"2024-09-14T03:34:02.519973+0000","last_unstale":"2024-09-14T03:34:56.915927+0000","last_undegraded":"2024-09-14T03:34:56.915927+0000","last_fullsized":"2024-09-14T03:34:56.915927+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:20.217955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831096+0000","last_change":"2024-09-14T03:33:59.399709+0000","last_active":"2024-09-14T03:34:56.831096+0000","last_peered":"2024-09-14T03:34:56.831096+0000","last_clean":"2024-09-14T03:34:56.831096+0000","last_became_active":"2024-09-14T03:33:59.398533+0000","last_became_peered":"2024-09-14T03:33:59.398533+0000","last_unstale":"2024-09-14T03:34:56.831096+0000","last_undegraded":"2024-09-14T03:34:56.831096+0000","last_fullsized":"2024-09-14T03:34:56.831096+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:12:43.525175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.917039+0000","last_change":"2024-09-14T03:34:02.520629+0000","last_active":"2024-09-14T03:34:56.917039+0000","last_peered":"2024-09-14T03:34:56.917039+0000","last_clean":"2024-09-14T03:34:56.917039+0000","last_became_active":"2024-09-14T03:34:02.520365+0000","last_became_peered":"2024-09-14T03:34:02.520365+0000","last_unstale":"2024-09-14T03:34:56.917039+0000","last_undegraded":"2024-09-14T03:34:56.917039+0000","last_fullsized":"2024-09-14T03:34:56.917039+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:29.519783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830800+0000","last_change":"2024-09-14T03:33:59.404858+0000","last_active":"2024-09-14T03:34:56.830800+0000","last_peered":"2024-09-14T03:34:56.830800+0000","last_clean":"2024-09-14T03:34:56.830800+0000","last_became_active":"2024-09-14T03:33:59.404261+0000","last_became_peered":"2024-09-14T03:33:59.404261+0000","last_unstale":"2024-09-14T03:34:56.830800+0000","last_undegraded":"2024-09-14T03:34:56.830800+0000","last_fullsized":"2024-09-14T03:34:56.830800+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:24:38.834576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916965+0000","last_change":"2024-09-14T03:34:02.516816+0000","last_active":"2024-09-14T03:34:56.916965+0000","last_peered":"2024-09-14T03:34:56.916965+0000","last_clean":"2024-09-14T03:34:56.916965+0000","last_became_active":"2024-09-14T03:34:02.516527+0000","last_became_peered":"2024-09-14T03:34:02.516527+0000","last_unstale":"2024-09-14T03:34:56.916965+0000","last_undegraded":"2024-09-14T03:34:56.916965+0000","last_fullsized":"2024-09-14T03:34:56.916965+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:35.627973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837514+0000","last_change":"2024-09-14T03:33:59.395532+0000","last_active":"2024-09-14T03:34:56.837514+0000","last_peered":"2024-09-14T03:34:56.837514+0000","last_clean":"2024-09-14T03:34:56.837514+0000","last_became_active":"2024-09-14T03:33:59.395255+0000","last_became_peered":"2024-09-14T03:33:59.395255+0000","last_unstale":"2024-09-14T03:34:56.837514+0000","last_undegraded":"2024-09-14T03:34:56.837514+0000","last_fullsized":"2024-09-14T03:34:56.837514+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:51.477913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839317+0000","last_change":"2024-09-14T03:34:02.520744+0000","last_active":"2024-09-14T03:34:56.839317+0000","last_peered":"2024-09-14T03:34:56.839317+0000","last_clean":"2024-09-14T03:34:56.839317+0000","last_became_active":"2024-09-14T03:34:02.520420+0000","last_became_peered":"2024-09-14T03:34:02.520420+0000","last_unstale":"2024-09-14T03:34:56.839317+0000","last_undegraded":"2024-09-14T03:34:56.839317+0000","last_fullsized":"2024-09-14T03:34:56.839317+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:32:43.573943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830896+0000","last_change":"2024-09-14T03:33:59.399299+0000","last_active":"2024-09-14T03:34:56.830896+0000","last_peered":"2024-09-14T03:34:56.830896+0000","last_clean":"2024-09-14T03:34:56.830896+0000","last_became_active":"2024-09-14T03:33:59.399008+0000","last_became_peered":"2024-09-14T03:33:59.399008+0000","last_unstale":"2024-09-14T03:34:56.830896+0000","last_undegraded":"2024-09-14T03:34:56.830896+0000","last_fullsized":"2024-09-14T03:34:56.830896+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:16:14.935721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916805+0000","last_change":"2024-09-14T03:34:02.521443+0000","last_active":"2024-09-14T03:34:56.916805+0000","last_peered":"2024-09-14T03:34:56.916805+0000","last_clean":"2024-09-14T03:34:56.916805+0000","last_became_active":"2024-09-14T03:34:02.519164+0000","last_became_peered":"2024-09-14T03:34:02.519164+0000","last_unstale":"2024-09-14T03:34:56.916805+0000","last_undegraded":"2024-09-14T03:34:56.916805+0000","last_fullsized":"2024-09-14T03:34:56.916805+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:28.199628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.293111+0000","last_change":"2024-09-14T03:34:56.917557+0000","last_active":"2024-09-14T03:35:05.293111+0000","last_peered":"2024-09-14T03:35:05.293111+0000","last_clean":"2024-09-14T03:35:05.293111+0000","last_became_active":"2024-09-14T03:34:52.000396+0000","last_became_peered":"2024-09-14T03:34:52.000396+0000","last_unstale":"2024-09-14T03:35:05.293111+0000","last_undegraded":"2024-09-14T03:35:05.293111+0000","last_fullsized":"2024-09-14T03:35:05.293111+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:18:00.378690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030274300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"62'79","reported_seq":58,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916460+0000","last_change":"2024-09-14T03:33:36.972974+0000","last_active":"2024-09-14T03:34:56.916460+0000","last_peered":"2024-09-14T03:34:56.916460+0000","last_clean":"2024-09-14T03:34:56.916460+0000","last_became_active":"2024-09-14T03:33:36.967695+0000","last_became_peered":"2024-09-14T03:33:36.967695+0000","last_unstale":"2024-09-14T03:34:56.916460+0000","last_undegraded":"2024-09-14T03:34:56.916460+0000","last_fullsized":"2024-09-14T03:34:56.916460+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_clean_scrub_stamp":"2024-09-14T03:31:39.719769+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:28:24.258573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839183+0000","last_change":"2024-09-14T03:33:59.406222+0000","last_active":"2024-09-14T03:34:56.839183+0000","last_peered":"2024-09-14T03:34:56.839183+0000","last_clean":"2024-09-14T03:34:56.839183+0000","last_became_active":"2024-09-14T03:33:59.405473+0000","last_became_peered":"2024-09-14T03:33:59.405473+0000","last_unstale":"2024-09-14T03:34:56.839183+0000","last_undegraded":"2024-09-14T03:34:56.839183+0000","last_fullsized":"2024-09-14T03:34:56.839183+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:26.326984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859862+0000","last_change":"2024-09-14T03:34:02.512416+0000","last_active":"2024-09-14T03:34:56.859862+0000","last_peered":"2024-09-14T03:34:56.859862+0000","last_clean":"2024-09-14T03:34:56.859862+0000","last_became_active":"2024-09-14T03:34:02.512135+0000","last_became_peered":"2024-09-14T03:34:02.512135+0000","last_unstale":"2024-09-14T03:34:56.859862+0000","last_undegraded":"2024-09-14T03:34:56.859862+0000","last_fullsized":"2024-09-14T03:34:56.859862+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:49.847882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:04.980477+0000","last_change":"2024-09-14T03:34:56.831437+0000","last_active":"2024-09-14T03:35:04.980477+0000","last_peered":"2024-09-14T03:35:04.980477+0000","last_clean":"2024-09-14T03:35:04.980477+0000","last_became_active":"2024-09-14T03:34:52.346558+0000","last_became_peered":"2024-09-14T03:34:52.346558+0000","last_unstale":"2024-09-14T03:35:04.980477+0000","last_undegraded":"2024-09-14T03:35:04.980477+0000","last_fullsized":"2024-09-14T03:35:04.980477+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:11.239970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050397699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916355+0000","last_change":"2024-09-14T03:33:59.409150+0000","last_active":"2024-09-14T03:34:56.916355+0000","last_peered":"2024-09-14T03:34:56.916355+0000","last_clean":"2024-09-14T03:34:56.916355+0000","last_became_active":"2024-09-14T03:33:59.408622+0000","last_became_peered":"2024-09-14T03:33:59.408622+0000","last_unstale":"2024-09-14T03:34:56.916355+0000","last_undegraded":"2024-09-14T03:34:56.916355+0000","last_fullsized":"2024-09-14T03:34:56.916355+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:11:03.508216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830380+0000","last_change":"2024-09-14T03:34:02.517068+0000","last_active":"2024-09-14T03:34:56.830380+0000","last_peered":"2024-09-14T03:34:56.830380+0000","last_clean":"2024-09-14T03:34:56.830380+0000","last_became_active":"2024-09-14T03:34:02.510158+0000","last_became_peered":"2024-09-14T03:34:02.510158+0000","last_unstale":"2024-09-14T03:34:56.830380+0000","last_undegraded":"2024-09-14T03:34:56.830380+0000","last_fullsized":"2024-09-14T03:34:56.830380+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:22.709122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.531485+0000","last_change":"2024-09-14T03:34:56.837898+0000","last_active":"2024-09-14T03:35:05.531485+0000","last_peered":"2024-09-14T03:35:05.531485+0000","last_clean":"2024-09-14T03:35:05.531485+0000","last_became_active":"2024-09-14T03:34:52.006563+0000","last_became_peered":"2024-09-14T03:34:52.006563+0000","last_unstale":"2024-09-14T03:35:05.531485+0000","last_undegraded":"2024-09-14T03:35:05.531485+0000","last_fullsized":"2024-09-14T03:35:05.531485+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:37.308715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027249499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830160+0000","last_change":"2024-09-14T03:33:59.399987+0000","last_active":"2024-09-14T03:34:56.830160+0000","last_peered":"2024-09-14T03:34:56.830160+0000","last_clean":"2024-09-14T03:34:56.830160+0000","last_became_active":"2024-09-14T03:33:59.399698+0000","last_became_peered":"2024-09-14T03:33:59.399698+0000","last_unstale":"2024-09-14T03:34:56.830160+0000","last_undegraded":"2024-09-14T03:34:56.830160+0000","last_fullsized":"2024-09-14T03:34:56.830160+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:58.806927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.836764+0000","last_change":"2024-09-14T03:34:02.515266+0000","last_active":"2024-09-14T03:34:56.836764+0000","last_peered":"2024-09-14T03:34:56.836764+0000","last_clean":"2024-09-14T03:34:56.836764+0000","last_became_active":"2024-09-14T03:34:02.515012+0000","last_became_peered":"2024-09-14T03:34:02.515012+0000","last_unstale":"2024-09-14T03:34:56.836764+0000","last_undegraded":"2024-09-14T03:34:56.836764+0000","last_fullsized":"2024-09-14T03:34:56.836764+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:52:23.510282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.531194+0000","last_change":"2024-09-14T03:34:56.837741+0000","last_active":"2024-09-14T03:35:05.531194+0000","last_peered":"2024-09-14T03:35:05.531194+0000","last_clean":"2024-09-14T03:35:05.531194+0000","last_became_active":"2024-09-14T03:34:52.346774+0000","last_became_peered":"2024-09-14T03:34:52.346774+0000","last_unstale":"2024-09-14T03:35:05.531194+0000","last_undegraded":"2024-09-14T03:35:05.531194+0000","last_fullsized":"2024-09-14T03:35:05.531194+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:53.687838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065452200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839454+0000","last_change":"2024-09-14T03:33:59.399736+0000","last_active":"2024-09-14T03:34:56.839454+0000","last_peered":"2024-09-14T03:34:56.839454+0000","last_clean":"2024-09-14T03:34:56.839454+0000","last_became_active":"2024-09-14T03:33:59.399434+0000","last_became_peered":"2024-09-14T03:33:59.399434+0000","last_unstale":"2024-09-14T03:34:56.839454+0000","last_undegraded":"2024-09-14T03:34:56.839454+0000","last_fullsized":"2024-09-14T03:34:56.839454+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:12.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830739+0000","last_change":"2024-09-14T03:34:02.507742+0000","last_active":"2024-09-14T03:34:56.830739+0000","last_peered":"2024-09-14T03:34:56.830739+0000","last_clean":"2024-09-14T03:34:56.830739+0000","last_became_active":"2024-09-14T03:34:02.507452+0000","last_became_peered":"2024-09-14T03:34:02.507452+0000","last_unstale":"2024-09-14T03:34:56.830739+0000","last_undegraded":"2024-09-14T03:34:56.830739+0000","last_fullsized":"2024-09-14T03:34:56.830739+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:50.407482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"67'2","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.530926+0000","last_change":"2024-09-14T03:34:56.837653+0000","last_active":"2024-09-14T03:35:05.530926+0000","last_peered":"2024-09-14T03:35:05.530926+0000","last_clean":"2024-09-14T03:35:05.530926+0000","last_became_active":"2024-09-14T03:34:52.005925+0000","last_became_peered":"2024-09-14T03:34:52.005925+0000","last_unstale":"2024-09-14T03:35:05.530926+0000","last_undegraded":"2024-09-14T03:35:05.530926+0000","last_fullsized":"2024-09-14T03:35:05.530926+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:57:41.107409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083349699999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837358+0000","last_change":"2024-09-14T03:33:59.400498+0000","last_active":"2024-09-14T03:34:56.837358+0000","last_peered":"2024-09-14T03:34:56.837358+0000","last_clean":"2024-09-14T03:34:56.837358+0000","last_became_active":"2024-09-14T03:33:59.400204+0000","last_became_peered":"2024-09-14T03:33:59.400204+0000","last_unstale":"2024-09-14T03:34:56.837358+0000","last_undegraded":"2024-09-14T03:34:56.837358+0000","last_fullsized":"2024-09-14T03:34:56.837358+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:07:48.735887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839417+0000","last_change":"2024-09-14T03:34:02.513933+0000","last_active":"2024-09-14T03:34:56.839417+0000","last_peered":"2024-09-14T03:34:56.839417+0000","last_clean":"2024-09-14T03:34:56.839417+0000","last_became_active":"2024-09-14T03:34:02.513689+0000","last_became_peered":"2024-09-14T03:34:02.513689+0000","last_unstale":"2024-09-14T03:34:56.839417+0000","last_undegraded":"2024-09-14T03:34:56.839417+0000","last_fullsized":"2024-09-14T03:34:56.839417+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:23.791135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"65'1","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:04.979871+0000","last_change":"2024-09-14T03:34:56.831590+0000","last_active":"2024-09-14T03:35:04.979871+0000","last_peered":"2024-09-14T03:35:04.979871+0000","last_clean":"2024-09-14T03:35:04.979871+0000","last_became_active":"2024-09-14T03:34:52.005945+0000","last_became_peered":"2024-09-14T03:34:52.005945+0000","last_unstale":"2024-09-14T03:35:04.979871+0000","last_undegraded":"2024-09-14T03:35:04.979871+0000","last_fullsized":"2024-09-14T03:35:04.979871+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:00.810273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058187600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830345+0000","last_change":"2024-09-14T03:34:34.954630+0000","last_active":"2024-09-14T03:34:56.830345+0000","last_peered":"2024-09-14T03:34:56.830345+0000","last_clean":"2024-09-14T03:34:56.830345+0000","last_became_active":"2024-09-14T03:34:34.954209+0000","last_became_peered":"2024-09-14T03:34:34.954209+0000","last_unstale":"2024-09-14T03:34:56.830345+0000","last_undegraded":"2024-09-14T03:34:56.830345+0000","last_fullsized":"2024-09-14T03:34:56.830345+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:57.878910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837283+0000","last_change":"2024-09-14T03:34:02.513000+0000","last_active":"2024-09-14T03:34:56.837283+0000","last_peered":"2024-09-14T03:34:56.837283+0000","last_clean":"2024-09-14T03:34:56.837283+0000","last_became_active":"2024-09-14T03:34:02.512679+0000","last_became_peered":"2024-09-14T03:34:02.512679+0000","last_unstale":"2024-09-14T03:34:56.837283+0000","last_undegraded":"2024-09-14T03:34:56.837283+0000","last_fullsized":"2024-09-14T03:34:56.837283+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:43:48.821069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.857050+0000","last_change":"2024-09-14T03:34:56.860310+0000","last_active":"2024-09-14T03:35:05.857050+0000","last_peered":"2024-09-14T03:35:05.857050+0000","last_clean":"2024-09-14T03:35:05.857050+0000","last_became_active":"2024-09-14T03:34:51.997457+0000","last_became_peered":"2024-09-14T03:34:51.997457+0000","last_unstale":"2024-09-14T03:35:05.857050+0000","last_undegraded":"2024-09-14T03:35:05.857050+0000","last_fullsized":"2024-09-14T03:35:05.857050+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:30.409340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070668399999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837791+0000","last_change":"2024-09-14T03:33:59.397629+0000","last_active":"2024-09-14T03:34:56.837791+0000","last_peered":"2024-09-14T03:34:56.837791+0000","last_clean":"2024-09-14T03:34:56.837791+0000","last_became_active":"2024-09-14T03:33:59.397377+0000","last_became_peered":"2024-09-14T03:33:59.397377+0000","last_unstale":"2024-09-14T03:34:56.837791+0000","last_undegraded":"2024-09-14T03:34:56.837791+0000","last_fullsized":"2024-09-14T03:34:56.837791+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:42:02.914679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.860019+0000","last_change":"2024-09-14T03:34:02.517598+0000","last_active":"2024-09-14T03:34:56.860019+0000","last_peered":"2024-09-14T03:34:56.860019+0000","last_clean":"2024-09-14T03:34:56.860019+0000","last_became_active":"2024-09-14T03:34:02.517367+0000","last_became_peered":"2024-09-14T03:34:02.517367+0000","last_unstale":"2024-09-14T03:34:56.860019+0000","last_undegraded":"2024-09-14T03:34:56.860019+0000","last_fullsized":"2024-09-14T03:34:56.860019+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:44.727080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:35:05.145236+0000","last_change":"2024-09-14T03:34:56.830903+0000","last_active":"2024-09-14T03:35:05.145236+0000","last_peered":"2024-09-14T03:35:05.145236+0000","last_clean":"2024-09-14T03:35:05.145236+0000","last_became_active":"2024-09-14T03:34:52.006158+0000","last_became_peered":"2024-09-14T03:34:52.006158+0000","last_unstale":"2024-09-14T03:35:05.145236+0000","last_undegraded":"2024-09-14T03:35:05.145236+0000","last_fullsized":"2024-09-14T03:35:05.145236+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:03.846292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059832600000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916770+0000","last_change":"2024-09-14T03:33:59.405301+0000","last_active":"2024-09-14T03:34:56.916770+0000","last_peered":"2024-09-14T03:34:56.916770+0000","last_clean":"2024-09-14T03:34:56.916770+0000","last_became_active":"2024-09-14T03:33:59.404994+0000","last_became_peered":"2024-09-14T03:33:59.404994+0000","last_unstale":"2024-09-14T03:34:56.916770+0000","last_undegraded":"2024-09-14T03:34:56.916770+0000","last_fullsized":"2024-09-14T03:34:56.916770+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:52:19.415253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830839+0000","last_change":"2024-09-14T03:34:02.516570+0000","last_active":"2024-09-14T03:34:56.830839+0000","last_peered":"2024-09-14T03:34:56.830839+0000","last_clean":"2024-09-14T03:34:56.830839+0000","last_became_active":"2024-09-14T03:34:02.516212+0000","last_became_peered":"2024-09-14T03:34:02.516212+0000","last_unstale":"2024-09-14T03:34:56.830839+0000","last_undegraded":"2024-09-14T03:34:56.830839+0000","last_fullsized":"2024-09-14T03:34:56.830839+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:33:23.282986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916200+0000","last_change":"2024-09-14T03:33:59.408805+0000","last_active":"2024-09-14T03:34:56.916200+0000","last_peered":"2024-09-14T03:34:56.916200+0000","last_clean":"2024-09-14T03:34:56.916200+0000","last_became_active":"2024-09-14T03:33:59.408521+0000","last_became_peered":"2024-09-14T03:33:59.408521+0000","last_unstale":"2024-09-14T03:34:56.916200+0000","last_undegraded":"2024-09-14T03:34:56.916200+0000","last_fullsized":"2024-09-14T03:34:56.916200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:20:00.847563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.830664+0000","last_change":"2024-09-14T03:34:02.511411+0000","last_active":"2024-09-14T03:34:56.830664+0000","last_peered":"2024-09-14T03:34:56.830664+0000","last_clean":"2024-09-14T03:34:56.830664+0000","last_became_active":"2024-09-14T03:34:02.511131+0000","last_became_peered":"2024-09-14T03:34:02.511131+0000","last_unstale":"2024-09-14T03:34:56.830664+0000","last_undegraded":"2024-09-14T03:34:56.830664+0000","last_fullsized":"2024-09-14T03:34:56.830664+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:23.033860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.836906+0000","last_change":"2024-09-14T03:33:59.400483+0000","last_active":"2024-09-14T03:34:56.836906+0000","last_peered":"2024-09-14T03:34:56.836906+0000","last_clean":"2024-09-14T03:34:56.836906+0000","last_became_active":"2024-09-14T03:33:59.400172+0000","last_became_peered":"2024-09-14T03:33:59.400172+0000","last_unstale":"2024-09-14T03:34:56.836906+0000","last_undegraded":"2024-09-14T03:34:56.836906+0000","last_fullsized":"2024-09-14T03:34:56.836906+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:40.756478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859865+0000","last_change":"2024-09-14T03:34:02.505712+0000","last_active":"2024-09-14T03:34:56.859865+0000","last_peered":"2024-09-14T03:34:56.859865+0000","last_clean":"2024-09-14T03:34:56.859865+0000","last_became_active":"2024-09-14T03:34:02.505460+0000","last_became_peered":"2024-09-14T03:34:02.505460+0000","last_unstale":"2024-09-14T03:34:56.859865+0000","last_undegraded":"2024-09-14T03:34:56.859865+0000","last_fullsized":"2024-09-14T03:34:56.859865+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:20.752579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.837257+0000","last_change":"2024-09-14T03:33:59.405640+0000","last_active":"2024-09-14T03:34:56.837257+0000","last_peered":"2024-09-14T03:34:56.837257+0000","last_clean":"2024-09-14T03:34:56.837257+0000","last_became_active":"2024-09-14T03:33:59.405382+0000","last_became_peered":"2024-09-14T03:33:59.405382+0000","last_unstale":"2024-09-14T03:34:56.837257+0000","last_undegraded":"2024-09-14T03:34:56.837257+0000","last_fullsized":"2024-09-14T03:34:56.837257+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:36.350128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859846+0000","last_change":"2024-09-14T03:34:02.516581+0000","last_active":"2024-09-14T03:34:56.859846+0000","last_peered":"2024-09-14T03:34:56.859846+0000","last_clean":"2024-09-14T03:34:56.859846+0000","last_became_active":"2024-09-14T03:34:02.516335+0000","last_became_peered":"2024-09-14T03:34:02.516335+0000","last_unstale":"2024-09-14T03:34:56.859846+0000","last_undegraded":"2024-09-14T03:34:56.859846+0000","last_fullsized":"2024-09-14T03:34:56.859846+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:06.070599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916927+0000","last_change":"2024-09-14T03:33:59.408573+0000","last_active":"2024-09-14T03:34:56.916927+0000","last_peered":"2024-09-14T03:34:56.916927+0000","last_clean":"2024-09-14T03:34:56.916927+0000","last_became_active":"2024-09-14T03:33:59.408264+0000","last_became_peered":"2024-09-14T03:33:59.408264+0000","last_unstale":"2024-09-14T03:34:56.916927+0000","last_undegraded":"2024-09-14T03:34:56.916927+0000","last_fullsized":"2024-09-14T03:34:56.916927+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:30.937449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916037+0000","last_change":"2024-09-14T03:34:02.516408+0000","last_active":"2024-09-14T03:34:56.916037+0000","last_peered":"2024-09-14T03:34:56.916037+0000","last_clean":"2024-09-14T03:34:56.916037+0000","last_became_active":"2024-09-14T03:34:02.516119+0000","last_became_peered":"2024-09-14T03:34:02.516119+0000","last_unstale":"2024-09-14T03:34:56.916037+0000","last_undegraded":"2024-09-14T03:34:56.916037+0000","last_fullsized":"2024-09-14T03:34:56.916037+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:38:16.570920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839391+0000","last_change":"2024-09-14T03:34:02.512557+0000","last_active":"2024-09-14T03:34:56.839391+0000","last_peered":"2024-09-14T03:34:56.839391+0000","last_clean":"2024-09-14T03:34:56.839391+0000","last_became_active":"2024-09-14T03:34:02.512183+0000","last_became_peered":"2024-09-14T03:34:02.512183+0000","last_unstale":"2024-09-14T03:34:56.839391+0000","last_undegraded":"2024-09-14T03:34:56.839391+0000","last_fullsized":"2024-09-14T03:34:56.839391+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:05:14.073579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916723+0000","last_change":"2024-09-14T03:33:59.406333+0000","last_active":"2024-09-14T03:34:56.916723+0000","last_peered":"2024-09-14T03:34:56.916723+0000","last_clean":"2024-09-14T03:34:56.916723+0000","last_became_active":"2024-09-14T03:33:59.406065+0000","last_became_peered":"2024-09-14T03:33:59.406065+0000","last_unstale":"2024-09-14T03:34:56.916723+0000","last_undegraded":"2024-09-14T03:34:56.916723+0000","last_fullsized":"2024-09-14T03:34:56.916723+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:04.009889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.839294+0000","last_change":"2024-09-14T03:34:02.508363+0000","last_active":"2024-09-14T03:34:56.839294+0000","last_peered":"2024-09-14T03:34:56.839294+0000","last_clean":"2024-09-14T03:34:56.839294+0000","last_became_active":"2024-09-14T03:34:02.508086+0000","last_became_peered":"2024-09-14T03:34:02.508086+0000","last_unstale":"2024-09-14T03:34:56.839294+0000","last_undegraded":"2024-09-14T03:34:56.839294+0000","last_fullsized":"2024-09-14T03:34:56.839294+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:47.194036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.916839+0000","last_change":"2024-09-14T03:33:59.406945+0000","last_active":"2024-09-14T03:34:56.916839+0000","last_peered":"2024-09-14T03:34:56.916839+0000","last_clean":"2024-09-14T03:34:56.916839+0000","last_became_active":"2024-09-14T03:33:59.406662+0000","last_became_peered":"2024-09-14T03:33:59.406662+0000","last_unstale":"2024-09-14T03:34:56.916839+0000","last_undegraded":"2024-09-14T03:34:56.916839+0000","last_fullsized":"2024-09-14T03:34:56.916839+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:46:42.749128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831327+0000","last_change":"2024-09-14T03:34:02.510922+0000","last_active":"2024-09-14T03:34:56.831327+0000","last_peered":"2024-09-14T03:34:56.831327+0000","last_clean":"2024-09-14T03:34:56.831327+0000","last_became_active":"2024-09-14T03:34:02.510663+0000","last_became_peered":"2024-09-14T03:34:02.510663+0000","last_unstale":"2024-09-14T03:34:56.831327+0000","last_undegraded":"2024-09-14T03:34:56.831327+0000","last_fullsized":"2024-09-14T03:34:56.831327+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:00:02.779787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.860081+0000","last_change":"2024-09-14T03:33:59.398992+0000","last_active":"2024-09-14T03:34:56.860081+0000","last_peered":"2024-09-14T03:34:56.860081+0000","last_clean":"2024-09-14T03:34:56.860081+0000","last_became_active":"2024-09-14T03:33:59.398725+0000","last_became_peered":"2024-09-14T03:33:59.398725+0000","last_unstale":"2024-09-14T03:34:56.860081+0000","last_undegraded":"2024-09-14T03:34:56.860081+0000","last_fullsized":"2024-09-14T03:34:56.860081+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:50:20.745864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.859899+0000","last_change":"2024-09-14T03:34:02.514810+0000","last_active":"2024-09-14T03:34:56.859899+0000","last_peered":"2024-09-14T03:34:56.859899+0000","last_clean":"2024-09-14T03:34:56.859899+0000","last_became_active":"2024-09-14T03:34:02.514500+0000","last_became_peered":"2024-09-14T03:34:02.514500+0000","last_unstale":"2024-09-14T03:34:56.859899+0000","last_undegraded":"2024-09-14T03:34:56.859899+0000","last_fullsized":"2024-09-14T03:34:56.859899+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:20:21.110046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":28,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:34:56.831482+0000","last_change":"2024-09-14T03:33:59.397898+0000","last_active":"2024-09-14T03:34:56.831482+0000","last_peered":"2024-09-14T03:34:56.831482+0000","last_clean":"2024-09-14T03:34:56.831482+0000","last_became_active":"2024-09-14T03:33:59.396670+0000","last_became_peered":"2024-09-14T03:33:59.396670+0000","last_unstale":"2024-09-14T03:34:56.831482+0000","last_undegraded":"2024-09-14T03:34:56.831482+0000","last_fullsized":"2024-09-14T03:34:56.831482+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:46.652243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397534,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":903104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.504,"15min":0.50800000000000001},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63900000000000001,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.495},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.52700000000000002,"15min":0.53500000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.64300000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.62},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.58299999999999996,"15min":0.59499999999999997},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67400000000000004,"5min":1.044,"15min":1.044},"last":0.47899999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.57699999999999996,"15min":0.59099999999999997},"min":{"1min":0.36099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.65100000000000002,"5min":1.012,"15min":1.012},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.54400000000000004,"15min":0.55600000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64100000000000001,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.54500000000000004}]},{"osd":3,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.56399999999999995,"15min":0.57799999999999996},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.67300000000000004,"5min":1.115,"15min":1.115},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.57699999999999996,"15min":0.59099999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.69199999999999995,"5min":0.87,"15min":0.87},"last":0.59199999999999997}]},{"osd":4,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.46999999999999997,"15min":0.47799999999999998},"min":{"1min":0.309,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60399999999999998,"5min":1.079,"15min":1.079},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.51900000000000002,"15min":0.53100000000000003},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.70599999999999996,"5min":1.282,"15min":1.282},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.51500000000000001,"15min":0.53100000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68600000000000005,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.437},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.52200000000000002,"15min":0.52900000000000003},"min":{"1min":0.35599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.71799999999999997,"5min":1.135,"15min":1.135},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.58399999999999996,"15min":0.59799999999999998},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.75,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.58399999999999996,"15min":0.59999999999999998},"min":{"1min":0.37,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73599999999999999,"5min":1.625,"15min":1.625},"last":0.60799999999999998}]}]},{"osd":6,"up_from":43,"seq":184683593762,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":904539,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.374,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.52200000000000002,"15min":0.51900000000000002},"min":{"1min":0.42199999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.55700000000000005}]},{"osd":1,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51100000000000001,"15min":0.50900000000000001},"min":{"1min":0.38900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48299999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.629},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.52700000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54900000000000004,"15min":0.55400000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72399999999999998,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.505}]},{"osd":4,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.442,"15min":0.44500000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.441,"5min":0.45900000000000002,"15min":0.46200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.45600000000000002,"15min":0.45900000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.59899999999999998,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.48299999999999998,"15min":0.48699999999999999},"min":{"1min":0.32700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60299999999999998,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 14 03:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.47499999999999998,"15min":0.47099999999999997},"min":{"1min":0.33500000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.72999999999999998,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.51500000000000001,"15min":0.51100000000000001},"min":{"1min":0.34699999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.49199999999999999}]}]},{"osd":5,"up_from":37,"seq":158913789991,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":314036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.61699999999999999,"15min":0.55700000000000005},"min":{"1min":0.41899999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":11.026999999999999,"5min":11.026999999999999,"15min":11.026999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.52000000000000002,"15min":0.52200000000000002},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64400000000000002,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.67300000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.51000000000000001,"15min":0.51400000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.64400000000000002,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.52200000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.51200000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.61899999999999999,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.505,"15min":0.50700000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.68200000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52700000000000002,"15min":0.52800000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.437}]},{"osd":4,"last update":"Sat Sep 14 03:34:50 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.28899999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.376,"15min":0.36399999999999999},"min":{"1min":0.32900000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:35:17 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.46200000000000002,"15min":0.45800000000000002},"min":{"1min":0.30199999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.55400000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:35:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.505,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.501,"5min":0.48499999999999999,"15min":0.48199999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.68400000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.622}]}]},{"osd":4,"up_from":31,"seq":133143986220,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":312859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.499,"15min":0.50700000000000001},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63100000000000001,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.499,"5min":0.51200000000000001,"15min":0.51600000000000001},"min":{"1min":0.35799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.625,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.64500000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:35:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.52600000000000002,"15min":0.53800000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68000000000000005,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.501,"5min":0.51300000000000001,"15min":0.51500000000000001},"min":{"1min":0.39100000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.68000000000000005,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.49399999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:35:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52900000000000003,"15min":0.53500000000000003},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.65700000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.52200000000000002,"15min":0.53600000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.63900000000000001,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.57599999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:35:29 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.54500000000000004,"15min":0.55700000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.63700000000000001,"5min":0.752,"15min":0.752},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.379,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.64300000000000002,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.68600000000000005}]},{"osd":5,"last update":"Sat Sep 14 03:35:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.39700000000000002,"15min":0.36099999999999999},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.65000000000000002,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.38900000000000001,"15min":0.36499999999999999},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.56599999999999995,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.51700000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:35:18 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.40600000000000003,"15min":0.40000000000000002},"min":{"1min":0.32800000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.45000000000000001,"15min":0.443},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 14 03:35:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.71699999999999997,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.70799999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.72099999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149744,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":314590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:11 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.35499999999999998,"15min":0.35799999999999998},"min":{"1min":0.23100000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.41199999999999998,"15min":0.42399999999999999},"min":{"1min":0.30099999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:35:11 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.379,"15min":0.38700000000000001},"min":{"1min":0.27400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.51100000000000001,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.379,"5min":0.38500000000000001,"15min":0.39300000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.30399999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:35:11 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.41499999999999998,"15min":0.42899999999999999},"min":{"1min":0.29399999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39400000000000002,"15min":0.40300000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.35499999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.56599999999999995,"15min":0.57299999999999995},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.379},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.502}]},{"osd":5,"last update":"Sat Sep 14 03:35:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.42199999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55300000000000005,"15min":0.55200000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:35:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54200000000000004,"15min":0.53900000000000003},"min":{"1min":0.41199999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.55100000000000005,"15min":0.54600000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.41199999999999998,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.627}]}]},{"osd":2,"up_from":18,"seq":77309411381,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":312144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.318,"5min":0.38,"15min":0.39800000000000002},"min":{"1min":0.20300000000000001,"5min":0.16300000000000001,"15min":0.16300000000000001},"max":{"1min":0.54700000000000004,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.376},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.40799999999999997,"15min":0.42299999999999999},"min":{"1min":0.22800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51600000000000001,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.35799999999999998,"15min":0.372},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.41599999999999998,"5min":0.621,"15min":0.621},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.315,"5min":0.378,"15min":0.38800000000000001},"min":{"1min":0.23899999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.436,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:35:07 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.40100000000000002,"15min":0.41099999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.46100000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55900000000000005,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.251}]},{"osd":4,"last update":"Sat Sep 14 03:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.38500000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.71899999999999997,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.437}]},{"osd":5,"last update":"Sat Sep 14 03:34:51 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56999999999999995,"15min":0.57099999999999995},"min":{"1min":0.45900000000000002,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.436,"5min":0.435,"15min":0.435},"max":{"1min":0.68500000000000005,"5min":0.745,"15min":0.745},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:35:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.57099999999999995,"15min":0.58099999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.67000000000000004,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.58999999999999997,"15min":0.60499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.499}]},{"osd":7,"last update":"Sat Sep 14 03:35:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54800000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.66200000000000003,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.436,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.48699999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574905,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":311378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:35:26 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34300000000000003,"15min":0.32000000000000001},"min":{"1min":0.24199999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.42499999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.32700000000000001,"5min":0.34399999999999997,"15min":0.33300000000000002},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.47399999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.36499999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.371,"15min":0.39000000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.46000000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.39200000000000002,"15min":0.40600000000000003},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.40999999999999998,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 14 03:35:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.437,"15min":0.45300000000000001},"min":{"1min":0.29199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.47199999999999998,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.442,"15min":0.46000000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.499,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.375}]},{"osd":4,"last update":"Sat Sep 14 03:35:32 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.56999999999999995,"15min":0.57499999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60399999999999998,"15min":0.60999999999999999},"min":{"1min":0.42699999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.64900000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:34:55 2024","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":0.79000000000000004,"15min":0.66500000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":15.747999999999999,"5min":15.747999999999999,"15min":15.747999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.5649999999999999,"5min":0.77000000000000002,"15min":0.63800000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":15.760999999999999,"5min":15.760999999999999,"15min":15.760999999999999},"last":0.67800000000000005}]},{"osd":6,"last update":"Sat Sep 14 03:35:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.446,"5min":0.89000000000000001,"15min":0.79800000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":15.775,"5min":15.775,"15min":15.775},"last":0.74199999999999999}]},{"osd":7,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36899999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.86199999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.752},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.361,"15min":1.486},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.746,"5min":15.734,"15min":15.734},"last":0.66900000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738429,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298632,"kb_used_data":1276,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446520,"statfs":{"total":95995035648,"available":95689236480,"internally_reserved":0,"allocated":1306624,"data_stored":903228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:35:26 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.317},"min":{"1min":0.23999999999999999,"5min":0.192,"15min":0.192},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.254},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.33200000000000002},"min":{"1min":0.27000000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:35:43 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.371,"15min":0.35899999999999999},"min":{"1min":0.254,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.318},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.38300000000000001,"15min":0.374},"min":{"1min":0.27600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.374}]},{"osd":3,"last update":"Sat Sep 14 03:35:08 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.42099999999999999,"15min":0.41999999999999998},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.44400000000000001,"15min":0.45400000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.33100000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.59899999999999998,"15min":0.60499999999999998},"min":{"1min":0.41899999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.1799999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.55600000000000005,"15min":0.55400000000000005},"min":{"1min":0.40300000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:35:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.55100000000000005,"15min":0.54100000000000004},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.74199999999999999,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59199999999999997,"15min":0.58799999999999997},"min":{"1min":0.42399999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.59699999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:35:12 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.53800000000000003,"15min":0.52500000000000002},"min":{"1min":0.441,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.63900000000000001,"15min":0.63600000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.2210000000000001,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57899999999999996,"15min":0.57699999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.57099999999999995,"15min":0.56399999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.52000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:35:56.840 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T03:35:56.840 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:35:56.840 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:35:56.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-14T03:35:57.436 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:35:57.561 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:57 smithi110 ceph-mon[35374]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:57.562 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:57 smithi110 systemd[1]: Stopping Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:35:57.562 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:57 smithi110 ceph-mon[27365]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:57.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:57 smithi195 ceph-mon[30695]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:58.446 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[66835]: ts=2024-09-14T03:35:58.156Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:35:58.446 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:58 smithi110 podman[71226]: 2024-09-14 03:35:58.17725483 +0000 UTC m=+0.114544984 container died 730197a8cb7fab1ddcf8f8d60ede85030d594968e270e347aff3e4f8c1854f01 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:58.791 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:58 smithi110 podman[71226]: 2024-09-14 03:35:58.448581408 +0000 UTC m=+0.385871550 container cleanup 730197a8cb7fab1ddcf8f8d60ede85030d594968e270e347aff3e4f8c1854f01 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:58.791 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:58 smithi110 bash[71226]: ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:35:59.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:35:59 smithi110 ceph-mon[35374]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:59.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:35:59 smithi110 ceph-mon[27365]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:59.356 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:59 smithi110 podman[71239]: 2024-09-14 03:35:59.103077243 +0000 UTC m=+0.921441930 container remove 730197a8cb7fab1ddcf8f8d60ede85030d594968e270e347aff3e4f8c1854f01 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:35:59.356 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:59 smithi110 podman[71239]: 2024-09-14 03:35:59.187095606 +0000 UTC m=+1.005460294 volume remove 8461e20ed19acae1a42fe3c4f218cdbceb38d4825dc64e11600c88721552c124 2024-09-14T03:35:59.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:35:59 smithi195 ceph-mon[30695]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:35:59.874 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:35:59.874 INFO:teuthology.orchestra.run.smithi110.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:35:59.946 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:59 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:35:59.946 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:59 smithi110 systemd[1]: Stopped Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:35:59.946 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:59 smithi110 systemd[1]: Starting Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:36:00.266 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:59 smithi110 podman[71377]: 2024-09-14 03:35:59.856441101 +0000 UTC m=+0.016604700 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-14T03:36:00.266 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:35:59 smithi110 podman[71377]: 2024-09-14 03:35:59.967720115 +0000 UTC m=+0.127883716 volume create e776c50bf85ba8dd69f38367de98d766fe91dd0d640dd7c0bfa25ba6e0e69030 2024-09-14T03:36:00.266 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 podman[71377]: 2024-09-14 03:36:00.068269688 +0000 UTC m=+0.228433329 container create 764997e49f35d23edf747b6ce67f2dab482d406113349b9f5709722242ad63be (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:00 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1566455109' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:36:00.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:36:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:36:00.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1566455109' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:36:00.517 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1566455109' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:36:00.517 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 podman[71377]: 2024-09-14 03:36:00.388149414 +0000 UTC m=+0.548313013 container init 764997e49f35d23edf747b6ce67f2dab482d406113349b9f5709722242ad63be (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:00.517 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 podman[71377]: 2024-09-14 03:36:00.390975797 +0000 UTC m=+0.551139394 container start 764997e49f35d23edf747b6ce67f2dab482d406113349b9f5709722242ad63be (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:00.517 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:00.403Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:00.403Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:00.404Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.110 port=9094 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:00.405Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:00.428Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:00.428Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:00.431Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:00.431Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 bash[71377]: 764997e49f35d23edf747b6ce67f2dab482d406113349b9f5709722242ad63be 2024-09-14T03:36:00.518 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:00 smithi110 systemd[1]: Started Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:36:01.119 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:36:01.119 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:36:01.377 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:01.395 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:01 smithi110 ceph-mon[35374]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:01.395 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:01 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:01.395 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:01 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:01.395 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:01 smithi110 ceph-mon[27365]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:01.395 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:01 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:01.395 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:01 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:01 smithi195 ceph-mon[30695]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:01 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:01 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:02.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:02 smithi110 ceph-mon[27365]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:36:02.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:02 smithi110 ceph-mon[27365]: Reconfiguring daemon prometheus.a on smithi195 2024-09-14T03:36:02.447 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:02.406Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000532051s 2024-09-14T03:36:02.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:02 smithi110 ceph-mon[35374]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:36:02.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:02 smithi110 ceph-mon[35374]: Reconfiguring daemon prometheus.a on smithi195 2024-09-14T03:36:02.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:02 smithi195 ceph-mon[30695]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:36:02.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:02 smithi195 ceph-mon[30695]: Reconfiguring daemon prometheus.a on smithi195 2024-09-14T03:36:03.233 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:36:03.233 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24310,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.195:6833/1180864653","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6832","nonce":1180864653},{"type":"v1","addr":"172.21.15.195:6833","nonce":1180864653}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:34:04.160374+0000","modified":"2024-09-14T03:34:15.891707+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24311},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24311":{"gid":24311,"name":"a","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.110:6835/2015946497","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6834","nonce":2015946497},{"type":"v1","addr":"172.21.15.110:6835","nonce":2015946497}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-14T03:36:03.233 INFO:teuthology.orchestra.run.smithi110.stderr:dumped fsmap epoch 7 2024-09-14T03:36:03.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:03 smithi110 ceph-mon[35374]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:03.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:03 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1866855086' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:03.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:03 smithi110 ceph-mon[27365]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:03.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:03 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1866855086' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:03.687 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:03 smithi195 ceph-mon[30695]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:03.687 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:03 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1866855086' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:03.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:03 smithi195 systemd[1]: Stopping Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:36:03.965 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:36:04.211 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:04.425 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.123Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:36:04.425 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.123Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:36:04.425 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.123Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:36:04.425 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.123Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:36:04.425 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.123Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:36:04.425 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.123Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:36:04.426 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.123Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:36:04.426 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.123Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:36:04.426 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.124Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:36:04.426 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.124Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:36:04.426 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.124Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:36:04.426 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[58676]: ts=2024-09-14T03:36:04.124Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:36:04.426 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 podman[61764]: 2024-09-14 03:36:04.133988251 +0000 UTC m=+0.147195651 container died d8f68feeab7845fedbd24e39f4db634d506f83f2be8d8d17b187599ea6ef45da (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:04.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 podman[61764]: 2024-09-14 03:36:04.531526687 +0000 UTC m=+0.544734086 container cleanup d8f68feeab7845fedbd24e39f4db634d506f83f2be8d8d17b187599ea6ef45da (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:04.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 bash[61764]: ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:36:05.158 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 podman[61776]: 2024-09-14 03:36:04.732924762 +0000 UTC m=+0.601458476 container remove d8f68feeab7845fedbd24e39f4db634d506f83f2be8d8d17b187599ea6ef45da (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:05.158 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:36:05.159 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 systemd[1]: Stopped Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:36:05.159 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.023s CPU time. 2024-09-14T03:36:05.159 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:04 smithi195 systemd[1]: Starting Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:36:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-mon[30695]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:05.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 podman[61869]: 2024-09-14 03:36:05.061360129 +0000 UTC m=+0.016729215 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:36:05.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 podman[61869]: 2024-09-14 03:36:05.202734448 +0000 UTC m=+0.158103522 container create 4460d604b8f635d23319035366e36f29a7d0d214dd140f1e48a564b451d0ebb9 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:05.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-mon[35374]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:05.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:05.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-mon[27365]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:05.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:05.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:05] ENGINE Bus STOPPING 2024-09-14T03:36:05.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:36:05.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:05] ENGINE Bus STOPPED 2024-09-14T03:36:05.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:05] ENGINE Bus STARTING 2024-09-14T03:36:05.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:05] ENGINE Serving on http://:::9283 2024-09-14T03:36:05.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:05] ENGINE Bus STARTED 2024-09-14T03:36:05.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:05] ENGINE Bus STOPPING 2024-09-14T03:36:05.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 podman[61869]: 2024-09-14 03:36:05.486287517 +0000 UTC m=+0.441656590 container init 4460d604b8f635d23319035366e36f29a7d0d214dd140f1e48a564b451d0ebb9 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:05.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 podman[61869]: 2024-09-14 03:36:05.4893094 +0000 UTC m=+0.444678476 container start 4460d604b8f635d23319035366e36f29a7d0d214dd140f1e48a564b451d0ebb9 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:36:05.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.511Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:36:05.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.511Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:36:05.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.511Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi195 (none))" 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.511Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.511Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.513Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.513Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.514Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.514Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.516Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.516Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.94µs 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.516Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.517Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.517Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-14T03:36:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.517Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.722µs wal_replay_duration=538.387µs wbl_replay_duration=91ns total_replay_duration=590.07µs 2024-09-14T03:36:05.961 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.518Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:36:05.961 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.518Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:36:05.961 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.518Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:36:05.961 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.532Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.661809ms db_storage=653ns remote_storage=1.096µs web_handler=348ns query_engine=544ns scrape=593.04µs scrape_sd=27.21µs notify=18.928µs notify_sd=6.273µs rules=13.80453ms tracing=5.833µs 2024-09-14T03:36:05.961 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.532Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:36:05.961 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:36:05.532Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:36:05.961 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 bash[61869]: 4460d604b8f635d23319035366e36f29a7d0d214dd140f1e48a564b451d0ebb9 2024-09-14T03:36:05.961 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:36:05 smithi195 systemd[1]: Started Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:36:06.070 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:36:06.070 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24310,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.195:6833/1180864653","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6832","nonce":1180864653},{"type":"v1","addr":"172.21.15.195:6833","nonce":1180864653}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:34:04.160374+0000","modified":"2024-09-14T03:34:15.891707+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24311},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24311":{"gid":24311,"name":"a","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.110:6835/2015946497","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6834","nonce":2015946497},{"type":"v1","addr":"172.21.15.110:6835","nonce":2015946497}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-14T03:36:06.071 INFO:teuthology.orchestra.run.smithi110.stderr:dumped fsmap epoch 7 2024-09-14T03:36:06.421 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:36:06.422 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE Bus STOPPED 2024-09-14T03:36:06.422 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE Bus STARTING 2024-09-14T03:36:06.422 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE Serving on http://:::9283 2024-09-14T03:36:06.422 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE Bus STARTED 2024-09-14T03:36:06.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE Bus STOPPING 2024-09-14T03:36:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:36:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1649059550' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:36:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:06.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:06.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1649059550' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:36:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:36:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:36:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:36:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:06.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:06 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1649059550' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:07.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:36:07.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE Bus STOPPED 2024-09-14T03:36:07.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:06] ENGINE Bus STARTING 2024-09-14T03:36:07.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:07 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:07] ENGINE Serving on http://:::9283 2024-09-14T03:36:07.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:07 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:07] ENGINE Bus STARTED 2024-09-14T03:36:07.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:36:07.671 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:08.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:08 smithi110 ceph-mon[35374]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:08.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:08 smithi110 ceph-mon[27365]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:08.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:08 smithi195 ceph-mon[30695]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:10.059 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:36:10.059 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":68,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:35:04.975662+0000","last_up_change":"2024-09-14T03:33:34.960002+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:33:57.860054+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:34:01.436541+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:34:50.939554+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6803","nonce":1137454602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6805","nonce":1137454602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6809","nonce":1137454602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":1137454602},{"type":"v1","addr":"172.21.15.110:6807","nonce":1137454602}]},"public_addr":"172.21.15.110:6803/1137454602","cluster_addr":"172.21.15.110:6805/1137454602","heartbeat_back_addr":"172.21.15.110:6809/1137454602","heartbeat_front_addr":"172.21.15.110:6807/1137454602","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6811","nonce":3479666487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6813","nonce":3479666487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6817","nonce":3479666487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3479666487},{"type":"v1","addr":"172.21.15.110:6815","nonce":3479666487}]},"public_addr":"172.21.15.110:6811/3479666487","cluster_addr":"172.21.15.110:6813/3479666487","heartbeat_back_addr":"172.21.15.110:6817/3479666487","heartbeat_front_addr":"172.21.15.110:6815/3479666487","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6819","nonce":3061361173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6821","nonce":3061361173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6825","nonce":3061361173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3061361173},{"type":"v1","addr":"172.21.15.110:6823","nonce":3061361173}]},"public_addr":"172.21.15.110:6819/3061361173","cluster_addr":"172.21.15.110:6821/3061361173","heartbeat_back_addr":"172.21.15.110:6825/3061361173","heartbeat_front_addr":"172.21.15.110:6823/3061361173","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6827","nonce":3810786960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6829","nonce":3810786960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6833","nonce":3810786960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":3810786960},{"type":"v1","addr":"172.21.15.110:6831","nonce":3810786960}]},"public_addr":"172.21.15.110:6827/3810786960","cluster_addr":"172.21.15.110:6829/3810786960","heartbeat_back_addr":"172.21.15.110:6833/3810786960","heartbeat_front_addr":"172.21.15.110:6831/3810786960","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6801","nonce":3499292350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6803","nonce":3499292350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6807","nonce":3499292350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3499292350},{"type":"v1","addr":"172.21.15.195:6805","nonce":3499292350}]},"public_addr":"172.21.15.195:6801/3499292350","cluster_addr":"172.21.15.195:6803/3499292350","heartbeat_back_addr":"172.21.15.195:6807/3499292350","heartbeat_front_addr":"172.21.15.195:6805/3499292350","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6809","nonce":4151750662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6811","nonce":4151750662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6815","nonce":4151750662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":4151750662},{"type":"v1","addr":"172.21.15.195:6813","nonce":4151750662}]},"public_addr":"172.21.15.195:6809/4151750662","cluster_addr":"172.21.15.195:6811/4151750662","heartbeat_back_addr":"172.21.15.195:6815/4151750662","heartbeat_front_addr":"172.21.15.195:6813/4151750662","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6817","nonce":258227795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6819","nonce":258227795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6823","nonce":258227795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":258227795},{"type":"v1","addr":"172.21.15.195:6821","nonce":258227795}]},"public_addr":"172.21.15.195:6817/258227795","cluster_addr":"172.21.15.195:6819/258227795","heartbeat_back_addr":"172.21.15.195:6823/258227795","heartbeat_front_addr":"172.21.15.195:6821/258227795","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6825","nonce":284932944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6827","nonce":284932944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6831","nonce":284932944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":284932944},{"type":"v1","addr":"172.21.15.195:6829","nonce":284932944}]},"public_addr":"172.21.15.195:6825/284932944","cluster_addr":"172.21.15.195:6827/284932944","heartbeat_back_addr":"172.21.15.195:6831/284932944","heartbeat_front_addr":"172.21.15.195:6829/284932944","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.5","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.23","mappings":[{"from":0,"to":2}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/2214025689":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/4093221761":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2856593023":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/1769339162":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6800/1862102623":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6801/1862102623":"2024-09-15T03:34:49.809403+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:36:10.393 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:10 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:10.393 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:10 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:10.393 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:10 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4156306858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:36:10.393 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:10 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:10.393 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:10 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:10.393 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:10 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/4156306858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:36:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:10 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:10 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:10 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/4156306858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:36:10.696 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:36:10 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:36:10.409Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003362848s 2024-09-14T03:36:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:11 smithi110 ceph-mon[35374]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:11 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:11 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:11 smithi110 ceph-mon[27365]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:11 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:11 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:11.256 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:36:11.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:11 smithi195 ceph-mon[30695]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:11 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:11 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:11.514 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:12.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:12 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:12.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:12 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:12.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:12 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:12.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:12 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:12.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:12 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:12.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:12 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:12.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:12 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:12.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:12 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:12.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:12 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:12.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:12 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:12.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:12 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:12.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:12 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:13.426 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:36:13.426 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24310,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.195:6833/1180864653","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6832","nonce":1180864653},{"type":"v1","addr":"172.21.15.195:6833","nonce":1180864653}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:34:04.160374+0000","modified":"2024-09-14T03:34:15.891707+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24311},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24311":{"gid":24311,"name":"a","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.110:6835/2015946497","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6834","nonce":2015946497},{"type":"v1","addr":"172.21.15.110:6835","nonce":2015946497}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-14T03:36:13.426 INFO:teuthology.orchestra.run.smithi110.stderr:dumped fsmap epoch 7 2024-09-14T03:36:13.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:13 smithi195 ceph-mon[30695]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:13.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:13 smithi110 ceph-mon[35374]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:13.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:13 smithi110 ceph-mon[27365]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:14.177 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 7, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-09-14T03:34:04.160374+0000', 'modified': '2024-09-14T03:34:15.891707+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24311}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24311': {'gid': 24311, 'name': 'a', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 6, 'addr': '172.21.15.110:6835/2015946497', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.110:6834', 'nonce': 2015946497}, {'type': 'v1', 'addr': '172.21.15.110:6835', 'nonce': 2015946497}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138320759226367, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'standby_count_wanted': 0} 2024-09-14T03:36:14.177 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-09-14T03:36:14.178 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph tell mds.a status 2024-09-14T03:36:14.506 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:14 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/4052866020' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:14.534 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:14.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:14 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/4052866020' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:14.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:14 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4052866020' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:36:15.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:15 smithi110 ceph-mon[35374]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:15.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:15 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:15.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:15 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:15.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:15 smithi110 ceph-mon[27365]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:15.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:15 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:15.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:15 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:15.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:15 smithi195 ceph-mon[30695]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:15 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:15 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:16.275 INFO:teuthology.orchestra.run.smithi110.stderr:2024-09-14T03:36:16.274+0000 7fc985ffb640 0 client.14754 ms_handle_reset on v2:172.21.15.110:6834/2015946497 2024-09-14T03:36:16.294 INFO:teuthology.orchestra.run.smithi110.stderr:2024-09-14T03:36:16.292+0000 7fc985ffb640 0 client.14760 ms_handle_reset on v2:172.21.15.110:6834/2015946497 2024-09-14T03:36:16.294 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:36:16.294 INFO:teuthology.orchestra.run.smithi110.stdout: "cluster_fsid": "5517b266-7249-11ef-bceb-c7b262605968", 2024-09-14T03:36:16.294 INFO:teuthology.orchestra.run.smithi110.stdout: "whoami": 0, 2024-09-14T03:36:16.294 INFO:teuthology.orchestra.run.smithi110.stdout: "id": 24311, 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout: "want_state": "up:active", 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout: "state": "up:active", 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout: "fs_name": "cephfs", 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout: "rank_uptime": 131.78472666499999, 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout: "mdsmap_epoch": 7, 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout: "osdmap_epoch": 68, 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout: "osdmap_epoch_barrier": 58, 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout: "uptime": 149.21255932299999 2024-09-14T03:36:16.295 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:36:16.909 INFO:tasks.cephadm:Setup complete, yielding 2024-09-14T03:36:16.910 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:36:16.919 INFO:teuthology.task.print:**** done end installing quincy cephadm ... 2024-09-14T03:36:16.919 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:36:16.957 INFO:teuthology.task.print:**** done start cephadm.shell ceph config set mgr... 2024-09-14T03:36:16.957 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-14T03:36:16.995 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi110.front.sepia.ceph.com 2024-09-14T03:36:16.995 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2024-09-14T03:36:17.565 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:17.687 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:17 smithi110 ceph-mon[35374]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:17.687 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:17 smithi110 ceph-mon[27365]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:17.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:17 smithi195 ceph-mon[30695]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:19.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:19 smithi110 ceph-mon[35374]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:19.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:19 smithi110 ceph-mon[27365]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:19.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:19 smithi195 ceph-mon[30695]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:20.141 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:19] ENGINE Bus STOPPING 2024-09-14T03:36:20.141 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:36:20.141 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:20] ENGINE Bus STOPPED 2024-09-14T03:36:20.141 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:20] ENGINE Bus STARTING 2024-09-14T03:36:20.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:20] ENGINE Serving on http://:::9283 2024-09-14T03:36:20.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:36:20] ENGINE Bus STARTED 2024-09-14T03:36:20.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:20] "GET /metrics HTTP/1.1" 200 236502 "" "Prometheus/2.43.0" 2024-09-14T03:36:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:36:20.569 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:36:20.581 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2024-09-14T03:36:20.581 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:36:20.620 INFO:teuthology.task.print:**** done start telemetry quincy... 2024-09-14T03:36:20.620 INFO:teuthology.run_tasks:Running task workunit... 2024-09-14T03:36:20.662 INFO:tasks.workunit:Pulling workunits from ref 1c941a66c1b420259b8fc5ac19e792dbbf60aded 2024-09-14T03:36:20.663 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T03:36:20.663 DEBUG:teuthology.orchestra.run.smithi110:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:36:20.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:36:20.680 INFO:teuthology.orchestra.run.smithi110.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-14T03:36:20.681 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:36:20.737 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-14T03:36:20.737 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-14T03:36:20.792 INFO:tasks.workunit:timeout=3h 2024-09-14T03:36:20.792 INFO:tasks.workunit:cleanup=True 2024-09-14T03:36:20.793 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 1c941a66c1b420259b8fc5ac19e792dbbf60aded 2024-09-14T03:36:20.848 INFO:tasks.workunit.client.0.smithi110.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-14T03:36:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:20.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' 2024-09-14T03:36:20.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:20.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:20.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:20.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:20.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:36:20.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:22 smithi110 ceph-mon[35374]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:22 smithi110 ceph-mon[35374]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-14T03:36:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:22 smithi110 ceph-mon[27365]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:22 smithi110 ceph-mon[27365]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-14T03:36:23.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:22 smithi195 ceph-mon[30695]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:22 smithi195 ceph-mon[30695]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-14T03:36:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:24 smithi110 ceph-mon[35374]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:25.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:24 smithi110 ceph-mon[27365]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:25.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:24 smithi195 ceph-mon[30695]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:26 smithi110 ceph-mon[35374]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:26 smithi110 ceph-mon[27365]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:26 smithi195 ceph-mon[30695]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:29.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:28 smithi110 ceph-mon[27365]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:28 smithi110 ceph-mon[35374]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:28 smithi195 ceph-mon[30695]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:30.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:36:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:36:30.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:30] "GET /metrics HTTP/1.1" 200 236502 "" "Prometheus/2.43.0" 2024-09-14T03:36:31.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:30 smithi110 ceph-mon[35374]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:30 smithi110 ceph-mon[27365]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:31.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:30 smithi195 ceph-mon[30695]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:32 smithi110 ceph-mon[35374]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:32 smithi110 ceph-mon[27365]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:33.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:32 smithi195 ceph-mon[30695]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:36:35.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:34 smithi110 ceph-mon[35374]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:34 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:34 smithi110 ceph-mon[27365]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:34 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:34 smithi195 ceph-mon[30695]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:34 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:36 smithi110 ceph-mon[35374]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:36 smithi110 ceph-mon[27365]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:37.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:36 smithi195 ceph-mon[30695]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:38 smithi110 ceph-mon[35374]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:38 smithi110 ceph-mon[27365]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:38 smithi195 ceph-mon[30695]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:40.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:36:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:36:40.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:40 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:40] "GET /metrics HTTP/1.1" 200 236506 "" "Prometheus/2.43.0" 2024-09-14T03:36:41.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:40 smithi110 ceph-mon[35374]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:41.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:40 smithi110 ceph-mon[27365]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:41.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:40 smithi195 ceph-mon[30695]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:43.261 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:42 smithi195 ceph-mon[30695]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:42 smithi110 ceph-mon[35374]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:43.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:42 smithi110 ceph-mon[27365]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:45.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:44 smithi110 ceph-mon[35374]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:45.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:44 smithi110 ceph-mon[27365]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:45.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:44 smithi195 ceph-mon[30695]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:47.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:46 smithi110 ceph-mon[35374]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:47.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:46 smithi110 ceph-mon[27365]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:47.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:46 smithi195 ceph-mon[30695]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:49 smithi110 ceph-mon[35374]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:49.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:49 smithi110 ceph-mon[27365]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:49.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:49 smithi195 ceph-mon[30695]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:50.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:50 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:50.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:50 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:50.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:36:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:36:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:50 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:36:50.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:36:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:36:50] "GET /metrics HTTP/1.1" 200 236499 "" "Prometheus/2.43.0" 2024-09-14T03:36:51.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:51 smithi110 ceph-mon[27365]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:51.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:51 smithi110 ceph-mon[35374]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:51.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:51 smithi195 ceph-mon[30695]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:53.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:53 smithi110 ceph-mon[35374]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:53 smithi110 ceph-mon[27365]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:53.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:53 smithi195 ceph-mon[30695]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:55 smithi110 ceph-mon[35374]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:55 smithi110 ceph-mon[27365]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:55.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:55 smithi195 ceph-mon[30695]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:57.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:57 smithi110 ceph-mon[35374]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:57 smithi110 ceph-mon[27365]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:57.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:57 smithi195 ceph-mon[30695]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:36:59 smithi110 ceph-mon[35374]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:36:59 smithi110 ceph-mon[27365]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:36:59.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:36:59 smithi195 ceph-mon[30695]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:00.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:37:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:37:00.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:00] "GET /metrics HTTP/1.1" 200 236499 "" "Prometheus/2.43.0" 2024-09-14T03:37:01.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:01 smithi110 ceph-mon[35374]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:01.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:01 smithi110 ceph-mon[27365]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:01.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:01 smithi195 ceph-mon[30695]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:03 smithi110 ceph-mon[35374]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:03 smithi110 ceph-mon[27365]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:03.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:03 smithi195 ceph-mon[30695]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:05 smithi110 ceph-mon[35374]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:05 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:05 smithi110 ceph-mon[27365]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:05 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:05.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:05 smithi195 ceph-mon[30695]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:05 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:07.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:07 smithi110 ceph-mon[27365]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:07 smithi110 ceph-mon[35374]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:07.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:07 smithi195 ceph-mon[30695]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:09.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:09 smithi110 ceph-mon[35374]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:09.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:09 smithi110 ceph-mon[27365]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:09.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:09 smithi195 ceph-mon[30695]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:10.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:37:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:37:10.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:10 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:10] "GET /metrics HTTP/1.1" 200 236490 "" "Prometheus/2.43.0" 2024-09-14T03:37:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:11 smithi110 ceph-mon[35374]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:11.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:11 smithi110 ceph-mon[27365]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:11.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:11 smithi195 ceph-mon[30695]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:13.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:13 smithi110 ceph-mon[35374]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:13.333 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:13 smithi110 ceph-mon[27365]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:13 smithi195 ceph-mon[30695]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:15 smithi110 ceph-mon[35374]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:15 smithi110 ceph-mon[27365]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:15 smithi195 ceph-mon[30695]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:16.697 INFO:tasks.workunit.client.0.smithi110.stderr:Note: switching to '1c941a66c1b420259b8fc5ac19e792dbbf60aded'. 2024-09-14T03:37:16.697 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr:state without impacting any branches by switching back to a branch. 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr: git switch -c 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr:Or undo this operation with: 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr: git switch - 2024-09-14T03:37:16.698 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:16.699 INFO:tasks.workunit.client.0.smithi110.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-14T03:37:16.699 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:16.699 INFO:tasks.workunit.client.0.smithi110.stderr:HEAD is now at 1c941a66c1b Merge pull request #54777 from cbodley/wip-rgw-meta-handlers 2024-09-14T03:37:16.700 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-14T03:37:16.764 INFO:tasks.workunit.client.0.smithi110.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T03:37:16.766 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T03:37:16.766 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T03:37:17.445 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T03:37:17.445 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:17 smithi110 ceph-mon[35374]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:17.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:17 smithi110 ceph-mon[27365]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:17.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:17 smithi195 ceph-mon[30695]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:17.519 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T03:37:17.560 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T03:37:17.561 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T03:37:17.561 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T03:37:17.600 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T03:37:17.603 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:37:17.603 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-14T03:37:17.659 INFO:tasks.workunit:Running workunits matching test_telemetry_quincy.sh on client.0... 2024-09-14T03:37:17.661 INFO:tasks.workunit:Running workunit test_telemetry_quincy.sh... 2024-09-14T03:37:17.661 DEBUG:teuthology.orchestra.run.smithi110:workunit test test_telemetry_quincy.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=1c941a66c1b420259b8fc5ac19e792dbbf60aded TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/test_telemetry_quincy.sh 2024-09-14T03:37:17.719 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph config set mgr mgr/telemetry/channel_ident true 2024-09-14T03:37:18.073 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph config set mgr mgr/telemetry/organization ceph-qa 2024-09-14T03:37:18.372 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:17 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:17] ENGINE Bus STOPPING 2024-09-14T03:37:18.373 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:37:18.373 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE Bus STOPPED 2024-09-14T03:37:18.447 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph config set mgr mgr/telemetry/description 'upgrade test cluster' 2024-09-14T03:37:18.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE Bus STARTING 2024-09-14T03:37:18.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE Serving on http://:::9283 2024-09-14T03:37:18.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE Bus STARTED 2024-09-14T03:37:18.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE Bus STOPPING 2024-09-14T03:37:18.806 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph telemetry preview 2024-09-14T03:37:19.245 INFO:tasks.workunit.client.0.smithi110.stdout:{ 2024-09-14T03:37:19.245 INFO:tasks.workunit.client.0.smithi110.stdout: "balancer": { 2024-09-14T03:37:19.245 INFO:tasks.workunit.client.0.smithi110.stdout: "active": true, 2024-09-14T03:37:19.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mode": "" 2024-09-14T03:37:19.245 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.245 INFO:tasks.workunit.client.0.smithi110.stdout: "channels": [ 2024-09-14T03:37:19.245 INFO:tasks.workunit.client.0.smithi110.stdout: "basic", 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "crash", 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "device", 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "ident" 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "channels_available": [ 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "basic", 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "ident", 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "crash", 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "device", 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "perf" 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "collections_available": [ 2024-09-14T03:37:19.246 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_base", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "device_base", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "crash_base", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "ident_base", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_perf", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_mds_metadata", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_usage", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_usage_by_class", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_rook_v01", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_memory_metrics", 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_options_bluestore" 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "collections_opted_in": [ 2024-09-14T03:37:19.247 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_base", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "device_base", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "crash_base", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "ident_base", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_perf", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_mds_metadata", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_usage", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_usage_by_class", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_rook_v01", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_memory_metrics", 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_options_bluestore" 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "config": { 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "active_changed": [ 2024-09-14T03:37:19.248 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_debug", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_service_ticket_ttl", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "bdev_debug_aio", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "client_check_pool_perm", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "client_mount_gid", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "client_mount_uid", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "client_quota", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "container_image", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "daemonize", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mgr", 2024-09-14T03:37:19.249 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mon", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_ms", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_osd", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_paxos", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "keyring", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_block_size", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_cache_size", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_compression", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_log", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_write_buffer_size", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_file", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_journald", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_stderr", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_allow_pool_delete", 2024-09-14T03:37:19.250 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_clock_drift_allowed", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_file", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_journald", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_stderr", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_data_avail_warn", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_election_default_strategy", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_host", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_max_pg_per_osd", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:37:19.251 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_bug", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_old_message", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "no_config_file", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_default_list", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_load_list", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_op_order", 2024-09-14T03:37:19.252 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_shutdown", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_profile", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-14T03:37:19.253 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_lim", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_res", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target_autotune", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue_cut_off", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recover_clone_overlap", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recovery_max_chunk", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_load_threshold", 2024-09-14T03:37:19.254 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_max_interval", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "public_addr", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "public_addrv", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "public_network", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd_default_features", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd_qos_exclude_ops", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "setgroup", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "setuser" 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster_changed": [ 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:37:19.255 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_debug", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_service_ticket_ttl", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "bdev_debug_aio", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "container_image", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mgr", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mon", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_ms", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_osd", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_paxos", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_file", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_journald", 2024-09-14T03:37:19.256 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_stderr", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_join_fs", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/allow_ptrace", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/container_init", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/migration_current", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/use_repo_digest", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-14T03:37:19.257 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/ssl_server_port", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/orchestrator/orchestrator", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/channel_ident", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/description", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/organization", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_allow_pool_delete", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_clock_drift_allowed", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_file", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_data_avail_warn", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_election_default_strategy", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_max_pg_per_osd", 2024-09-14T03:37:19.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:37:19.259 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_bug", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_old_message", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_default_list", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_load_list", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_op_order", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_shutdown", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_profile", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target", 2024-09-14T03:37:19.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target_autotune", 2024-09-14T03:37:19.261 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue", 2024-09-14T03:37:19.261 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue_cut_off", 2024-09-14T03:37:19.261 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:37:19.261 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:37:19.261 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recover_clone_overlap", 2024-09-14T03:37:19.261 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recovery_max_chunk", 2024-09-14T03:37:19.261 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_load_threshold", 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_max_interval", 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "public_network", 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_cache_enabled", 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_enable_ops_log", 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_enable_usage_log" 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "contact": null, 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "crashes": [], 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "created": "2024-09-14T03:28:36.548032Z", 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "crush": { 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_algs": { 2024-09-14T03:37:19.262 INFO:tasks.workunit.client.0.smithi110.stdout: "straw2": 3 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_sizes": { 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: "2": 1, 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: "4": 2 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_types": { 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: "1": 2, 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: "11": 1 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: "compat_weight_set": false, 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: "device_classes": [ 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: 8 2024-09-14T03:37:19.263 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "num_buckets": 6, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "num_devices": 8, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "num_rules": 1, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "num_types": 12, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "num_weight_sets": 0, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "tunables": { 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "allowed_bucket_algs": 54, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_local_fallback_tries": 0, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_local_tries": 0, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_total_tries": 50, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_descend_once": 1, 2024-09-14T03:37:19.264 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_stable": 1, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_vary_r": 1, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v2_rules": 0, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v3_rules": 0, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v4_buckets": 1, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v5_rules": 0, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "legacy_tunables": 0, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "minimum_required_version": "jewel", 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "optimal_tunables": 1, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "profile": "jewel", 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables": 1, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables2": 1, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables3": 1, 2024-09-14T03:37:19.265 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables5": 1, 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "straw_calc_version": 1 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "description": "upgrade test cluster", 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "fs": { 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "feature_flags": { 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "enable_multiple": true, 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "ever_enabled_multiple": true 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "filesystems": [ 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "approx_ctime": "2024-09", 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "balancer_enabled": false, 2024-09-14T03:37:19.266 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_caps": 0, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_dns": 10, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_inos": 13, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_subtrees": 2, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "ever_allowed_features": 0, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "explicitly_allowed_features": 0, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "files": 0, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "max_mds": 1, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_data_pools": 1, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_in": 1, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_mds": 1, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": 0, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_standby_replay": 0, 2024-09-14T03:37:19.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_up": 1, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "snaps": 0, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "standby_count_wanted": 0 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_standby_mds": 1, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "total_num_mds": 2 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "hosts": { 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num": 2, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mds": 2, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mgr": 2, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mon": 2, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_osd": 2 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.268 INFO:tasks.workunit.client.0.smithi110.stdout: "leaderboard": false, 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "license": "sharing-1-0", 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "metadata": { 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 2 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.269 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 2 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 2 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 2 2024-09-14T03:37:19.270 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 3 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:19.271 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 3 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 3 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.272 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 3 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 8 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.273 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 8 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 8 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.274 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:19.275 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 8 2024-09-14T03:37:19.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.275 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_objectstore": { 2024-09-14T03:37:19.275 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": 8 2024-09-14T03:37:19.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.275 INFO:tasks.workunit.client.0.smithi110.stdout: "rotational": { 2024-09-14T03:37:19.275 INFO:tasks.workunit.client.0.smithi110.stdout: "0": 8 2024-09-14T03:37:19.276 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.276 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.276 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.276 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:19.276 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:19.276 INFO:tasks.workunit.client.0.smithi110.stdout: "features": { 2024-09-14T03:37:19.277 INFO:tasks.workunit.client.0.smithi110.stdout: "optional": [], 2024-09-14T03:37:19.277 INFO:tasks.workunit.client.0.smithi110.stdout: "persistent": [ 2024-09-14T03:37:19.277 INFO:tasks.workunit.client.0.smithi110.stdout: "kraken", 2024-09-14T03:37:19.277 INFO:tasks.workunit.client.0.smithi110.stdout: "luminous", 2024-09-14T03:37:19.277 INFO:tasks.workunit.client.0.smithi110.stdout: "mimic", 2024-09-14T03:37:19.277 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap-prune", 2024-09-14T03:37:19.278 INFO:tasks.workunit.client.0.smithi110.stdout: "nautilus", 2024-09-14T03:37:19.278 INFO:tasks.workunit.client.0.smithi110.stdout: "octopus", 2024-09-14T03:37:19.278 INFO:tasks.workunit.client.0.smithi110.stdout: "pacific", 2024-09-14T03:37:19.278 INFO:tasks.workunit.client.0.smithi110.stdout: "elector-pinging", 2024-09-14T03:37:19.278 INFO:tasks.workunit.client.0.smithi110.stdout: "quincy" 2024-09-14T03:37:19.278 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:19.279 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.279 INFO:tasks.workunit.client.0.smithi110.stdout: "ipv4_addr_mons": 6, 2024-09-14T03:37:19.279 INFO:tasks.workunit.client.0.smithi110.stdout: "ipv6_addr_mons": 0, 2024-09-14T03:37:19.279 INFO:tasks.workunit.client.0.smithi110.stdout: "min_mon_release": 17, 2024-09-14T03:37:19.279 INFO:tasks.workunit.client.0.smithi110.stdout: "v1_addr_mons": 3, 2024-09-14T03:37:19.279 INFO:tasks.workunit.client.0.smithi110.stdout: "v2_addr_mons": 3 2024-09-14T03:37:19.280 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.280 INFO:tasks.workunit.client.0.smithi110.stdout: "organization": "ceph-qa", 2024-09-14T03:37:19.280 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:19.280 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster_network": false, 2024-09-14T03:37:19.280 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:19.281 INFO:tasks.workunit.client.0.smithi110.stdout: "require_min_compat_client": "luminous", 2024-09-14T03:37:19.281 INFO:tasks.workunit.client.0.smithi110.stdout: "require_osd_release": "quincy" 2024-09-14T03:37:19.281 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.281 INFO:tasks.workunit.client.0.smithi110.stdout: "pools": [ 2024-09-14T03:37:19.282 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:19.282 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:19.282 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr" 2024-09-14T03:37:19.282 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.282 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:19.282 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:19.283 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:19.283 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:19.283 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:19.283 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 1, 2024-09-14T03:37:19.283 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 1, 2024-09-14T03:37:19.284 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 1, 2024-09-14T03:37:19.284 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:19.284 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:19.285 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115868426, 2024-09-14T03:37:19.285 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 1781760, 2024-09-14T03:37:19.285 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:19.285 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:19.285 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 1781760, 2024-09-14T03:37:19.285 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:19.286 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:19.286 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 2, 2024-09-14T03:37:19.286 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:19.286 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 2.450442480039783e-06, 2024-09-14T03:37:19.286 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:19.287 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:19.287 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 126, 2024-09-14T03:37:19.287 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 236544, 2024-09-14T03:37:19.287 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 590368, 2024-09-14T03:37:19.287 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 590368, 2024-09-14T03:37:19.287 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:19.288 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 1771104, 2024-09-14T03:37:19.288 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 113, 2024-09-14T03:37:19.288 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 1667072 2024-09-14T03:37:19.288 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.288 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:19.289 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:19.289 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:19.289 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.289 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:19.289 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:19.289 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 32, 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 32, 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 2, 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115868426, 2024-09-14T03:37:19.290 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 98304, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 98304, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 22, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 3410, 2024-09-14T03:37:19.291 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 3410, 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 10230, 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 51, 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 19456 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:19.292 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 64, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 64, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 3, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115868426, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 0, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:19.293 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 0.0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 0, 2024-09-14T03:37:19.294 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 0, 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 0, 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 0 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd" 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:19.295 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 8, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 8, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 4, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115868426, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 12288, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 12288, 2024-09-14T03:37:19.296 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 2, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 1.689964435058755e-08, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 19, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 19, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 57, 2024-09-14T03:37:19.297 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 2, 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 2048 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd": { 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: "mirroring_by_pool": [ 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: false 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_images_by_pool": [ 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:19.298 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pools": 1 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "report_id": "2dc8c2a9-6f51-4b3c-b455-44a7525f06ca", 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "report_timestamp": "2024-09-14T03:37:19.238216", 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "report_version": 1, 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "rook": { 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster": { 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "external-mode": null, 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "allow-multiple-per-node": null, 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "count": null, 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "max-id": null, 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "pvc": { 2024-09-14T03:37:19.299 INFO:tasks.workunit.client.0.smithi110.stdout: "enabled": null 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "stretch": { 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "enabled": null 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "network": { 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "provider": null 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "storage": { 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "device-set": { 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "non-portable": null, 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "portable": null, 2024-09-14T03:37:19.300 INFO:tasks.workunit.client.0.smithi110.stdout: "total": null 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: "csi": { 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: "kubernetes": { 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: "node": { 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: "kubernetes-total": null, 2024-09-14T03:37:19.301 INFO:tasks.workunit.client.0.smithi110.stdout: "with-ceph-daemons": null, 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-cephfs-plugin": null, 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-nfs-plugin": null, 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-rbd-plugin": null 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "usage": { 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "storage-class": { 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket": null, 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs": null, 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "nfs": null, 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd": null, 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: "total": null 2024-09-14T03:37:19.302 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "services": {}, 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "usage": { 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 105, 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "pools": 4, 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_by_class": { 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "ssd": { 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "total_avail_bytes": 765516918784, 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "total_bytes": 767960285184, 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_bytes": 2443366400, 2024-09-14T03:37:19.303 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_raw_bytes": 2443366400, 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_raw_ratio": 0.003181631211191416 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout: "total_avail_bytes": 765516918784, 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout: "total_bytes": 767960285184, 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_bytes": 2443366400 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout:} 2024-09-14T03:37:19.304 INFO:tasks.workunit.client.0.smithi110.stdout:Device report is generated separately. To see it run 'ceph telemetry preview-device'. 2024-09-14T03:37:19.305 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph telemetry preview-device 2024-09-14T03:37:19.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:37:19.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE Bus STOPPED 2024-09-14T03:37:19.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:18] ENGINE Bus STARTING 2024-09-14T03:37:19.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:19] ENGINE Serving on http://:::9283 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:19] ENGINE Bus STARTED 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:19] ENGINE Bus STOPPING 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/4172887366' entity='client.admin' 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2140793121' entity='client.admin' 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/4172887366' entity='client.admin' 2024-09-14T03:37:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' 2024-09-14T03:37:19.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2140793121' entity='client.admin' 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:18 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/4172887366' entity='client.admin' 2024-09-14T03:37:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' 2024-09-14T03:37:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2140793121' entity='client.admin' 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:19.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:18 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:19.716 INFO:tasks.workunit.client.0.smithi110.stdout:{ 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "a5230552-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "INTEL_SSDPED1D480GA_a523a9c6-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "20240914-033141": { 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "device": { 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "protocol": "NVMe", 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "nvme" 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "firmware_version": "E2010325", 2024-09-14T03:37:19.717 INFO:tasks.workunit.client.0.smithi110.stdout: "host_id": "a5230552-724a-11ef-a07b-0cc47a8fcc35", 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "json_format_version": [ 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "local_time": { 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "asctime": "Sat Sep 14 03:31:41 2024 UTC", 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "time_t": 1726284701 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "logical_block_size": 512, 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_controller_id": 0, 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:37:19.718 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_namespaces": [ 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "capacity": { 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "formatted_lba_size": 512, 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 1, 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "size": { 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "utilization": { 2024-09-14T03:37:19.719 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_pci_vendor": { 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 32902, 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "subsystem_id": 32902 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "Device stats": { 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_fw_downgrades": { 2024-09-14T03:37:19.720 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "crc_error_count": { 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "erase_fail_count": { 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.721 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "host_bytes_written": { 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 9651523 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "host_ctx_wear_used": { 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "media_bytes_read": { 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "nand_bytes_written": { 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_stat_indicator": { 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "pll_lock_loss_count": { 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.723 INFO:tasks.workunit.client.0.smithi110.stdout: "program_fail_count": { 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "soft_ecc_err_rate": { 2024-09-14T03:37:19.724 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: "thermal_throttle_status": { 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: "cnt": 0, 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: "pct": 0 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_host_reads": { 2024-09-14T03:37:19.725 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 14 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_media_wear": { 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": "0.000000" 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_timer": { 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 1145676 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.726 INFO:tasks.workunit.client.0.smithi110.stdout: "unexp_power_loss": { 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "wear_leveling": { 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "avg": 0, 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "max": 0, 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.727 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:37:19.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.728 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:37:19.728 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare": 100, 2024-09-14T03:37:19.728 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare_threshold": 0, 2024-09-14T03:37:19.728 INFO:tasks.workunit.client.0.smithi110.stdout: "controller_busy_time": 3744, 2024-09-14T03:37:19.728 INFO:tasks.workunit.client.0.smithi110.stdout: "critical_warning": 0, 2024-09-14T03:37:19.728 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_read": 130374666, 2024-09-14T03:37:19.728 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_written": 632522258, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "host_reads": 2081328881, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "host_writes": 14581181920, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "media_errors": 0, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_err_log_entries": 0, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "percentage_used": 0, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycles": 25695, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_hours": 30070, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": 32, 2024-09-14T03:37:19.729 INFO:tasks.workunit.client.0.smithi110.stdout: "unsafe_shutdowns": 8731 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_vendor": "intel", 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_version": { 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: "string": "<1.2", 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycle_count": 25695, 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_time": { 2024-09-14T03:37:19.730 INFO:tasks.workunit.client.0.smithi110.stdout: "hours": 30070 2024-09-14T03:37:19.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.731 INFO:tasks.workunit.client.0.smithi110.stdout: "serial_number": "deleted", 2024-09-14T03:37:19.731 INFO:tasks.workunit.client.0.smithi110.stdout: "smart_status": { 2024-09-14T03:37:19.731 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme": { 2024-09-14T03:37:19.731 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:19.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.731 INFO:tasks.workunit.client.0.smithi110.stdout: "passed": true 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl": { 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "argv": [ 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl", 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "-x", 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "--json=o", 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "/dev/nvme0n1" 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "build_info": "(local build)", 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "exit_status": 0, 2024-09-14T03:37:19.732 INFO:tasks.workunit.client.0.smithi110.stdout: "output": [ 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "Serial Number: deleted", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Version: E2010325", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller ID: 0", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "NVMe Version: <1.2", 2024-09-14T03:37:19.733 INFO:tasks.workunit.client.0.smithi110.stdout: "Number of Namespaces: 1", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Local Time is: Sat Sep 14 03:31:41 2024 UTC", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.734 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported Power States", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 512 0 2", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:37:19.735 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Critical Warning: 0x00", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Temperature: 32 Celsius", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare: 100%", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Percentage Used: 0%", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Read: 130,374,666 [66.7 TB]", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Written: 632,522,258 [323 TB]", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Read Commands: 2,081,328,881", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Write Commands: 14,581,181,920", 2024-09-14T03:37:19.736 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller Busy Time: 3,744", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "Power Cycles: 25,695", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "Power On Hours: 30,070", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "Unsafe Shutdowns: 8,731", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information Log Entries: 0", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "No Errors Logged", 2024-09-14T03:37:19.737 INFO:tasks.workunit.client.0.smithi110.stdout: "" 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: "svn_revision": "5155", 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: "version": [ 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": { 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: "current": 32 2024-09-14T03:37:19.738 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.739 INFO:tasks.workunit.client.0.smithi110.stdout: "user_capacity": { 2024-09-14T03:37:19.739 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:19.739 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:19.739 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.739 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.739 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.739 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.739 INFO:tasks.workunit.client.0.smithi110.stdout: "a524ce46-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "INTEL_SSDPED1D480GA_a52546be-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "20240914-033143": { 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "device": { 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "protocol": "NVMe", 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "nvme" 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "firmware_version": "E2010325", 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "host_id": "a524ce46-724a-11ef-a07b-0cc47a8fcc35", 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: "json_format_version": [ 2024-09-14T03:37:19.740 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "local_time": { 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "asctime": "Sat Sep 14 03:31:43 2024 UTC", 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "time_t": 1726284703 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "logical_block_size": 512, 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_controller_id": 0, 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_namespaces": [ 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:19.741 INFO:tasks.workunit.client.0.smithi110.stdout: "capacity": { 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "formatted_lba_size": 512, 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 1, 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "size": { 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "utilization": { 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:19.742 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_pci_vendor": { 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 32902, 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "subsystem_id": 32902 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "Device stats": { 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_fw_downgrades": { 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.743 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "crc_error_count": { 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "erase_fail_count": { 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "host_bytes_written": { 2024-09-14T03:37:19.744 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 10171674 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "host_ctx_wear_used": { 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "media_bytes_read": { 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "nand_bytes_written": { 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.745 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_stat_indicator": { 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "pll_lock_loss_count": { 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "program_fail_count": { 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.746 INFO:tasks.workunit.client.0.smithi110.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "soft_ecc_err_rate": { 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.747 INFO:tasks.workunit.client.0.smithi110.stdout: "thermal_throttle_status": { 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "cnt": 0, 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "pct": 0 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_host_reads": { 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 15 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_media_wear": { 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.748 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": "0.000000" 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_timer": { 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 1390641 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "unexp_power_loss": { 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "wear_leveling": { 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:19.749 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "avg": 0, 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "max": 0, 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare": 100, 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare_threshold": 0, 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "controller_busy_time": 3961, 2024-09-14T03:37:19.750 INFO:tasks.workunit.client.0.smithi110.stdout: "critical_warning": 0, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_read": 141031320, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_written": 666610881, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "host_reads": 2714455028, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "host_writes": 13997711582, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "media_errors": 0, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_err_log_entries": 0, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "percentage_used": 0, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycles": 35342, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_hours": 38021, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": 33, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "unsafe_shutdowns": 11022 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.751 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_vendor": "intel", 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_version": { 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "string": "<1.2", 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycle_count": 35342, 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_time": { 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "hours": 38021 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "serial_number": "deleted", 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "smart_status": { 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme": { 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:19.752 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "passed": true 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl": { 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "argv": [ 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl", 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "-x", 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "--json=o", 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "/dev/nvme0n1" 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "build_info": "(local build)", 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "exit_status": 0, 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "output": [ 2024-09-14T03:37:19.753 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "Serial Number: deleted", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Version: E2010325", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller ID: 0", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "NVMe Version: <1.2", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "Number of Namespaces: 1", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:37:19.754 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Local Time is: Sat Sep 14 03:31:43 2024 UTC", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported Power States", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:37:19.755 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 512 0 2", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "Critical Warning: 0x00", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "Temperature: 33 Celsius", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare: 100%", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "Percentage Used: 0%", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Read: 141,031,320 [72.2 TB]", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Written: 666,610,881 [341 TB]", 2024-09-14T03:37:19.756 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Read Commands: 2,714,455,028", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Write Commands: 13,997,711,582", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller Busy Time: 3,961", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "Power Cycles: 35,342", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "Power On Hours: 38,021", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "Unsafe Shutdowns: 11,022", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information Log Entries: 0", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "No Errors Logged", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "" 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:37:19.757 INFO:tasks.workunit.client.0.smithi110.stdout: "svn_revision": "5155", 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: "version": [ 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": { 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: "current": 33 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: "user_capacity": { 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.758 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.759 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:19.759 INFO:tasks.workunit.client.0.smithi110.stdout:} 2024-09-14T03:37:19.759 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph telemetry preview-all 2024-09-14T03:37:19.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:37:19.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:19] ENGINE Bus STOPPED 2024-09-14T03:37:19.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:19] ENGINE Bus STARTING 2024-09-14T03:37:19.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:19] ENGINE Serving on http://:::9283 2024-09-14T03:37:19.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:19] ENGINE Bus STARTED 2024-09-14T03:37:20.192 INFO:tasks.workunit.client.0.smithi110.stdout:{ 2024-09-14T03:37:20.192 INFO:tasks.workunit.client.0.smithi110.stdout: "device_report": { 2024-09-14T03:37:20.192 INFO:tasks.workunit.client.0.smithi110.stdout: "a5230552-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:20.192 INFO:tasks.workunit.client.0.smithi110.stdout: "INTEL_SSDPED1D480GA_a523a9c6-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "20240914-033141": { 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "device": { 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "protocol": "NVMe", 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "nvme" 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "firmware_version": "E2010325", 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "host_id": "a5230552-724a-11ef-a07b-0cc47a8fcc35", 2024-09-14T03:37:20.193 INFO:tasks.workunit.client.0.smithi110.stdout: "json_format_version": [ 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: "local_time": { 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: "asctime": "Sat Sep 14 03:31:41 2024 UTC", 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: "time_t": 1726284701 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: "logical_block_size": 512, 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:37:20.194 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_controller_id": 0, 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_namespaces": [ 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "capacity": { 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "formatted_lba_size": 512, 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 1, 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "size": { 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:20.195 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "utilization": { 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_pci_vendor": { 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 32902, 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "subsystem_id": 32902 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:37:20.196 INFO:tasks.workunit.client.0.smithi110.stdout: "Device stats": { 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_fw_downgrades": { 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "crc_error_count": { 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.197 INFO:tasks.workunit.client.0.smithi110.stdout: "erase_fail_count": { 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "host_bytes_written": { 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 9651523 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "host_ctx_wear_used": { 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "media_bytes_read": { 2024-09-14T03:37:20.198 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "nand_bytes_written": { 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_stat_indicator": { 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "pll_lock_loss_count": { 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "program_fail_count": { 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.200 INFO:tasks.workunit.client.0.smithi110.stdout: "soft_ecc_err_rate": { 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "thermal_throttle_status": { 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "cnt": 0, 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "pct": 0 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_host_reads": { 2024-09-14T03:37:20.201 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 14 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_media_wear": { 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": "0.000000" 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_timer": { 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 1145676 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "unexp_power_loss": { 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "wear_leveling": { 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "avg": 0, 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "max": 0, 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:37:20.203 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare": 100, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare_threshold": 0, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "controller_busy_time": 3744, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "critical_warning": 0, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_read": 130374666, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_written": 632522258, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "host_reads": 2081328881, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "host_writes": 14581181920, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "media_errors": 0, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "num_err_log_entries": 0, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "percentage_used": 0, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycles": 25695, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_hours": 30070, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": 32, 2024-09-14T03:37:20.204 INFO:tasks.workunit.client.0.smithi110.stdout: "unsafe_shutdowns": 8731 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_vendor": "intel", 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_version": { 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "string": "<1.2", 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycle_count": 25695, 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_time": { 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "hours": 30070 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "serial_number": "deleted", 2024-09-14T03:37:20.205 INFO:tasks.workunit.client.0.smithi110.stdout: "smart_status": { 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme": { 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "passed": true 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl": { 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "argv": [ 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl", 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "-x", 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "--json=o", 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "/dev/nvme0n1" 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.206 INFO:tasks.workunit.client.0.smithi110.stdout: "build_info": "(local build)", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "exit_status": 0, 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "output": [ 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "Serial Number: deleted", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Version: E2010325", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller ID: 0", 2024-09-14T03:37:20.207 INFO:tasks.workunit.client.0.smithi110.stdout: "NVMe Version: <1.2", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Number of Namespaces: 1", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Local Time is: Sat Sep 14 03:31:41 2024 UTC", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported Power States", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:37:20.208 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 512 0 2", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "Critical Warning: 0x00", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "Temperature: 32 Celsius", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare: 100%", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:37:20.209 INFO:tasks.workunit.client.0.smithi110.stdout: "Percentage Used: 0%", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Read: 130,374,666 [66.7 TB]", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Written: 632,522,258 [323 TB]", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Read Commands: 2,081,328,881", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Write Commands: 14,581,181,920", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller Busy Time: 3,744", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Power Cycles: 25,695", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Power On Hours: 30,070", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Unsafe Shutdowns: 8,731", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information Log Entries: 0", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "No Errors Logged", 2024-09-14T03:37:20.210 INFO:tasks.workunit.client.0.smithi110.stdout: "" 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: "svn_revision": "5155", 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: "version": [ 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": { 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: "current": 32 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: "user_capacity": { 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:20.211 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: "a524ce46-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: "INTEL_SSDPED1D480GA_a52546be-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: "20240914-033143": { 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: "device": { 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: "protocol": "NVMe", 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "nvme" 2024-09-14T03:37:20.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "firmware_version": "E2010325", 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "host_id": "a524ce46-724a-11ef-a07b-0cc47a8fcc35", 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "json_format_version": [ 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "local_time": { 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "asctime": "Sat Sep 14 03:31:43 2024 UTC", 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "time_t": 1726284703 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "logical_block_size": 512, 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_controller_id": 0, 2024-09-14T03:37:20.213 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_namespaces": [ 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "capacity": { 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "formatted_lba_size": 512, 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 1, 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "size": { 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.214 INFO:tasks.workunit.client.0.smithi110.stdout: "utilization": { 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_pci_vendor": { 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 32902, 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "subsystem_id": 32902 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "Device stats": { 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_fw_downgrades": { 2024-09-14T03:37:20.215 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "crc_error_count": { 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "erase_fail_count": { 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "host_bytes_written": { 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 10171674 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "host_ctx_wear_used": { 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "media_bytes_read": { 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "nand_bytes_written": { 2024-09-14T03:37:20.217 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_stat_indicator": { 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "pll_lock_loss_count": { 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "program_fail_count": { 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "soft_ecc_err_rate": { 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.219 INFO:tasks.workunit.client.0.smithi110.stdout: "thermal_throttle_status": { 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "cnt": 0, 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "pct": 0 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_host_reads": { 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 15 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_media_wear": { 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.220 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": "0.000000" 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_timer": { 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 1390641 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "unexp_power_loss": { 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "wear_leveling": { 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "avg": 0, 2024-09-14T03:37:20.221 INFO:tasks.workunit.client.0.smithi110.stdout: "max": 0, 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare": 100, 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare_threshold": 0, 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: "controller_busy_time": 3961, 2024-09-14T03:37:20.222 INFO:tasks.workunit.client.0.smithi110.stdout: "critical_warning": 0, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_read": 141031320, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_written": 666610881, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "host_reads": 2714455028, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "host_writes": 13997711582, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "media_errors": 0, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "num_err_log_entries": 0, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "percentage_used": 0, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycles": 35342, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_hours": 38021, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": 33, 2024-09-14T03:37:20.223 INFO:tasks.workunit.client.0.smithi110.stdout: "unsafe_shutdowns": 11022 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_vendor": "intel", 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_version": { 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "string": "<1.2", 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycle_count": 35342, 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_time": { 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "hours": 38021 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "serial_number": "deleted", 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "smart_status": { 2024-09-14T03:37:20.224 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme": { 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "passed": true 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl": { 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "argv": [ 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl", 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "-x", 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "--json=o", 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "/dev/nvme0n1" 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "build_info": "(local build)", 2024-09-14T03:37:20.225 INFO:tasks.workunit.client.0.smithi110.stdout: "exit_status": 0, 2024-09-14T03:37:20.226 INFO:tasks.workunit.client.0.smithi110.stdout: "output": [ 2024-09-14T03:37:20.226 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:37:20.226 INFO:tasks.workunit.client.0.smithi110.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:37:20.226 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.226 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:37:20.226 INFO:tasks.workunit.client.0.smithi110.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:37:20.226 INFO:tasks.workunit.client.0.smithi110.stdout: "Serial Number: deleted", 2024-09-14T03:37:20.226 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Version: E2010325", 2024-09-14T03:37:20.227 INFO:tasks.workunit.client.0.smithi110.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller ID: 0", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "NVMe Version: <1.2", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Number of Namespaces: 1", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Local Time is: Sat Sep 14 03:31:43 2024 UTC", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:37:20.228 INFO:tasks.workunit.client.0.smithi110.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported Power States", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 512 0 2", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:37:20.229 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Critical Warning: 0x00", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Temperature: 33 Celsius", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare: 100%", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Percentage Used: 0%", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Read: 141,031,320 [72.2 TB]", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Written: 666,610,881 [341 TB]", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Read Commands: 2,714,455,028", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Write Commands: 13,997,711,582", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller Busy Time: 3,961", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Power Cycles: 35,342", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Power On Hours: 38,021", 2024-09-14T03:37:20.230 INFO:tasks.workunit.client.0.smithi110.stdout: "Unsafe Shutdowns: 11,022", 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information Log Entries: 0", 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "No Errors Logged", 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "" 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "svn_revision": "5155", 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: "version": [ 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:20.231 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": { 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: "current": 33 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: "user_capacity": { 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.232 INFO:tasks.workunit.client.0.smithi110.stdout: "report": { 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "balancer": { 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "active": true, 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "mode": "" 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "channels": [ 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "basic", 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "crash", 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "device", 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "ident" 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "channels_available": [ 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "basic", 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "ident", 2024-09-14T03:37:20.233 INFO:tasks.workunit.client.0.smithi110.stdout: "crash", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "device", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "perf" 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "collections_available": [ 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_base", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "device_base", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "crash_base", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "ident_base", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_perf", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_mds_metadata", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_usage", 2024-09-14T03:37:20.234 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_usage_by_class", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_rook_v01", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_memory_metrics", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_options_bluestore" 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "collections_opted_in": [ 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_base", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "device_base", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "crash_base", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "ident_base", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_perf", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_mds_metadata", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_usage", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_usage_by_class", 2024-09-14T03:37:20.235 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_rook_v01", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_memory_metrics", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_options_bluestore" 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "config": { 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "active_changed": [ 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_debug", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_service_ticket_ttl", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "bdev_debug_aio", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "client_check_pool_perm", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "client_mount_gid", 2024-09-14T03:37:20.236 INFO:tasks.workunit.client.0.smithi110.stdout: "client_mount_uid", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "client_quota", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "container_image", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "daemonize", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mgr", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mon", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_ms", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_osd", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_paxos", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "keyring", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_block_size", 2024-09-14T03:37:20.237 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_cache_size", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_compression", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_log", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_write_buffer_size", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_file", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_journald", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_stderr", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_allow_pool_delete", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_clock_drift_allowed", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_file", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_journald", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_stderr", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_data_avail_warn", 2024-09-14T03:37:20.238 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_election_default_strategy", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_host", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_max_pg_per_osd", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:37:20.239 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_bug", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_old_message", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "no_config_file", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_default_list", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_load_list", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_op_order", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_shutdown", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:37:20.240 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_profile", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_lim", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_res", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-14T03:37:20.241 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target_autotune", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue_cut_off", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recover_clone_overlap", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recovery_max_chunk", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_load_threshold", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_max_interval", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "public_addr", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "public_addrv", 2024-09-14T03:37:20.242 INFO:tasks.workunit.client.0.smithi110.stdout: "public_network", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd_default_features", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd_qos_exclude_ops", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "setgroup", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "setuser" 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster_changed": [ 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_debug", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_service_ticket_ttl", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "bdev_debug_aio", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "container_image", 2024-09-14T03:37:20.243 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mgr", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mon", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_ms", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_osd", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_paxos", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_file", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_journald", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_stderr", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_join_fs", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/allow_ptrace", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/container_init", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/migration_current", 2024-09-14T03:37:20.244 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/use_repo_digest", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/ssl_server_port", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/orchestrator/orchestrator", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/channel_ident", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/description", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/organization", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_allow_pool_delete", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_clock_drift_allowed", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_file", 2024-09-14T03:37:20.245 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_data_avail_warn", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_election_default_strategy", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_max_pg_per_osd", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:37:20.246 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_bug", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_old_message", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_default_list", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_load_list", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_op_order", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_shutdown", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:37:20.247 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_profile", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target_autotune", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue_cut_off", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recover_clone_overlap", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recovery_max_chunk", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_load_threshold", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_max_interval", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:37:20.248 INFO:tasks.workunit.client.0.smithi110.stdout: "public_network", 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_cache_enabled", 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_enable_ops_log", 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_enable_usage_log" 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "contact": null, 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "crashes": [], 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "created": "2024-09-14T03:28:36.548032Z", 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "crush": { 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_algs": { 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "straw2": 3 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.249 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_sizes": { 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "2": 1, 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "4": 2 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_types": { 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "1": 2, 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "11": 1 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "compat_weight_set": false, 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "device_classes": [ 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: 8 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_buckets": 6, 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_devices": 8, 2024-09-14T03:37:20.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_rules": 1, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_types": 12, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_weight_sets": 0, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "tunables": { 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "allowed_bucket_algs": 54, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_local_fallback_tries": 0, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_local_tries": 0, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_total_tries": 50, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_descend_once": 1, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_stable": 1, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_vary_r": 1, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v2_rules": 0, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v3_rules": 0, 2024-09-14T03:37:20.251 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v4_buckets": 1, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v5_rules": 0, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "legacy_tunables": 0, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "minimum_required_version": "jewel", 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "optimal_tunables": 1, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "profile": "jewel", 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables": 1, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables2": 1, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables3": 1, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables5": 1, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "straw_calc_version": 1 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "description": "upgrade test cluster", 2024-09-14T03:37:20.252 INFO:tasks.workunit.client.0.smithi110.stdout: "fs": { 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "feature_flags": { 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "enable_multiple": true, 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "ever_enabled_multiple": true 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "filesystems": [ 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "approx_ctime": "2024-09", 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "balancer_enabled": false, 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_caps": 0, 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_dns": 10, 2024-09-14T03:37:20.253 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_inos": 13, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_subtrees": 2, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "ever_allowed_features": 0, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "explicitly_allowed_features": 0, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "files": 0, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "max_mds": 1, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "num_data_pools": 1, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "num_in": 1, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "num_mds": 1, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": 0, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "num_standby_replay": 0, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "num_up": 1, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "snaps": 0, 2024-09-14T03:37:20.254 INFO:tasks.workunit.client.0.smithi110.stdout: "standby_count_wanted": 0 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "num_standby_mds": 1, 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "total_num_mds": 2 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "hosts": { 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "num": 2, 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mds": 2, 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mgr": 2, 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mon": 2, 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_osd": 2 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "leaderboard": false, 2024-09-14T03:37:20.255 INFO:tasks.workunit.client.0.smithi110.stdout: "license": "sharing-1-0", 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "metadata": { 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 2 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:20.256 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 2 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 2 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-14T03:37:20.257 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 2 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 3 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-14T03:37:20.258 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 3 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 3 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:20.259 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 3 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 8 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-14T03:37:20.260 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 8 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 8 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 8 2024-09-14T03:37:20.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_objectstore": { 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": 8 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "rotational": { 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "0": 8 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "features": { 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "optional": [], 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "persistent": [ 2024-09-14T03:37:20.262 INFO:tasks.workunit.client.0.smithi110.stdout: "kraken", 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "luminous", 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "mimic", 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap-prune", 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "nautilus", 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "octopus", 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "pacific", 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "elector-pinging", 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "quincy" 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "ipv4_addr_mons": 6, 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "ipv6_addr_mons": 0, 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "min_mon_release": 17, 2024-09-14T03:37:20.263 INFO:tasks.workunit.client.0.smithi110.stdout: "v1_addr_mons": 3, 2024-09-14T03:37:20.264 INFO:tasks.workunit.client.0.smithi110.stdout: "v2_addr_mons": 3 2024-09-14T03:37:20.264 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.264 INFO:tasks.workunit.client.0.smithi110.stdout: "organization": "ceph-qa", 2024-09-14T03:37:20.264 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:20.264 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster_network": false, 2024-09-14T03:37:20.264 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:20.264 INFO:tasks.workunit.client.0.smithi110.stdout: "require_min_compat_client": "luminous", 2024-09-14T03:37:20.264 INFO:tasks.workunit.client.0.smithi110.stdout: "require_osd_release": "quincy" 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: "pools": [ 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr" 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:20.265 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 1, 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 1, 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 1, 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115868426, 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 1781760, 2024-09-14T03:37:20.266 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 1781760, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 2, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 2.450442480039783e-06, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:20.267 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 126, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 236544, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 590368, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 590368, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 1771104, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 113, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 1667072 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.268 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:20.269 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 32, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 32, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 2, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115868426, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 98304, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:20.270 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 98304, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 22, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 3410, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 3410, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:20.271 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 10230, 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 51, 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 19456 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:20.272 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 64, 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 64, 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 3, 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115868426, 2024-09-14T03:37:20.273 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 0.0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 0, 2024-09-14T03:37:20.274 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 0, 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 0, 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 0, 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 0 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd" 2024-09-14T03:37:20.275 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 8, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 8, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 4, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115868426, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 12288, 2024-09-14T03:37:20.276 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 12288, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 2, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 1.689964435058755e-08, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 19, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 19, 2024-09-14T03:37:20.277 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 57, 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 2, 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 2048 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.278 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd": { 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: "mirroring_by_pool": [ 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: false 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: "num_images_by_pool": [ 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pools": 1 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: "report_id": "2dc8c2a9-6f51-4b3c-b455-44a7525f06ca", 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: "report_timestamp": "2024-09-14T03:37:20.174626", 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: "report_version": 1, 2024-09-14T03:37:20.279 INFO:tasks.workunit.client.0.smithi110.stdout: "rook": { 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster": { 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "external-mode": null, 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "allow-multiple-per-node": null, 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "count": null, 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "max-id": null, 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "pvc": { 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "enabled": null 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "stretch": { 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: "enabled": null 2024-09-14T03:37:20.280 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: "network": { 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: "provider": null 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: "storage": { 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: "device-set": { 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: "non-portable": null, 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: "portable": null, 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: "total": null 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.281 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "csi": { 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "kubernetes": { 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "node": { 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "kubernetes-total": null, 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "with-ceph-daemons": null, 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-cephfs-plugin": null, 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-nfs-plugin": null, 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-rbd-plugin": null 2024-09-14T03:37:20.282 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "usage": { 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "storage-class": { 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket": null, 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs": null, 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "nfs": null, 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd": null, 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "total": null 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.283 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "services": {}, 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "usage": { 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 105, 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "pools": 4, 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_by_class": { 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "ssd": { 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "total_avail_bytes": 765516918784, 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "total_bytes": 767960285184, 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_bytes": 2443366400, 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_raw_bytes": 2443366400, 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_raw_ratio": 0.003181631211191416 2024-09-14T03:37:20.284 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.285 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:20.285 INFO:tasks.workunit.client.0.smithi110.stdout: "total_avail_bytes": 765516918784, 2024-09-14T03:37:20.285 INFO:tasks.workunit.client.0.smithi110.stdout: "total_bytes": 767960285184, 2024-09-14T03:37:20.285 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_bytes": 2443366400 2024-09-14T03:37:20.285 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.285 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:20.285 INFO:tasks.workunit.client.0.smithi110.stdout:} 2024-09-14T03:37:20.286 INFO:tasks.workunit.client.0.smithi110.stderr:++ ceph telemetry collection ls 2024-09-14T03:37:20.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:19 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:37:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:20.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:20.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:19 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:20.644 INFO:tasks.workunit.client.0.smithi110.stderr:+ COLLECTIONS='New collections are available: 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:['\''basic_base'\'', '\''basic_mds_metadata'\'', '\''basic_pool_options_bluestore'\'', '\''basic_pool_usage'\'', '\''basic_rook_v01'\'', '\''basic_usage_by_class'\'', '\''crash_base'\'', '\''device_base'\'', '\''ident_base'\'', '\''perf_memory_metrics'\'', '\''perf_perf'\''] 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:NAME STATUS DESC 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:37:20.645 INFO:tasks.workunit.client.0.smithi110.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster ' 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:+ NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics" "basic_pool_options_bluestore") 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:+ [[ New collections are available: 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:NAME STATUS DESC 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:37:20.646 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_perf* ]] 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:+ [[ New collections are available: 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:37:20.647 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:NAME STATUS DESC 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_mds_metadata* ]] 2024-09-14T03:37:20.648 INFO:tasks.workunit.client.0.smithi110.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:+ [[ New collections are available: 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:NAME STATUS DESC 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:37:20.649 INFO:tasks.workunit.client.0.smithi110.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_usage* ]] 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:+ [[ New collections are available: 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:NAME STATUS DESC 2024-09-14T03:37:20.650 INFO:tasks.workunit.client.0.smithi110.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:37:20.651 INFO:tasks.workunit.client.0.smithi110.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_rook_v01* ]] 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:+ [[ New collections are available: 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:NAME STATUS DESC 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:37:20.652 INFO:tasks.workunit.client.0.smithi110.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_memory_metrics* ]] 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:+ [[ New collections are available: 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:37:20.653 INFO:tasks.workunit.client.0.smithi110.stderr: 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:NAME STATUS DESC 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:37:20.654 INFO:tasks.workunit.client.0.smithi110.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:37:20.655 INFO:tasks.workunit.client.0.smithi110.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:37:20.655 INFO:tasks.workunit.client.0.smithi110.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_options_bluestore* ]] 2024-09-14T03:37:20.655 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph telemetry on --license sharing-1-0 2024-09-14T03:37:20.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:20] "GET /metrics HTTP/1.1" 200 236472 "" "Prometheus/2.43.0" 2024-09-14T03:37:21.102 INFO:tasks.workunit.client.0.smithi110.stdout:Telemetry is on. 2024-09-14T03:37:21.102 INFO:tasks.workunit.client.0.smithi110.stdout:Some channels are disabled, please enable with: 2024-09-14T03:37:21.102 INFO:tasks.workunit.client.0.smithi110.stdout:`ceph telemetry enable channel perf` 2024-09-14T03:37:21.113 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph telemetry enable channel perf 2024-09-14T03:37:21.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:20] ENGINE Bus STOPPING 2024-09-14T03:37:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:21.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:21.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:20 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:21.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:21.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:20 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:21.564 INFO:tasks.workunit.client.0.smithi110.stdout:channel_perf is enabled 2024-09-14T03:37:21.576 INFO:tasks.workunit.client.0.smithi110.stderr:++ ceph config get mgr mgr/telemetry/last_opt_revision 2024-09-14T03:37:21.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:37:21.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE Bus STOPPED 2024-09-14T03:37:21.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE Bus STARTING 2024-09-14T03:37:21.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE Serving on http://:::9283 2024-09-14T03:37:21.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE Bus STARTED 2024-09-14T03:37:21.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE Bus STOPPING 2024-09-14T03:37:21.921 INFO:tasks.workunit.client.0.smithi110.stderr:+ LAST_OPT_REVISION=1 2024-09-14T03:37:21.921 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 1 -ne 1 ']' 2024-09-14T03:37:21.921 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph telemetry show 2024-09-14T03:37:22.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:37:22.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE Bus STOPPED 2024-09-14T03:37:22.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:21] ENGINE Bus STARTING 2024-09-14T03:37:22.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:22] ENGINE Serving on http://:::9283 2024-09-14T03:37:22.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:37:22] ENGINE Bus STARTED 2024-09-14T03:37:22.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1161772441' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:37:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1161772441' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:37:22.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1161772441' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:37:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:21 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1161772441' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:37:22.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:22.463 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:22.463 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:22.463 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.463 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:22.463 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1161772441' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:37:22.463 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:21 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1161772441' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:37:22.560 INFO:tasks.workunit.client.0.smithi110.stdout:{ 2024-09-14T03:37:22.560 INFO:tasks.workunit.client.0.smithi110.stdout: "balancer": { 2024-09-14T03:37:22.560 INFO:tasks.workunit.client.0.smithi110.stdout: "active": true, 2024-09-14T03:37:22.560 INFO:tasks.workunit.client.0.smithi110.stdout: "mode": "" 2024-09-14T03:37:22.560 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.560 INFO:tasks.workunit.client.0.smithi110.stdout: "channels": [ 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "basic", 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "crash", 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "device", 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "ident", 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "perf" 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "channels_available": [ 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "basic", 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "ident", 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "crash", 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "device", 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: "perf" 2024-09-14T03:37:22.561 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "collections_available": [ 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_base", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "device_base", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "crash_base", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "ident_base", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_perf", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_mds_metadata", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_usage", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_usage_by_class", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_rook_v01", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_memory_metrics", 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_options_bluestore" 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.562 INFO:tasks.workunit.client.0.smithi110.stdout: "collections_opted_in": [ 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_base", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "device_base", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "crash_base", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "ident_base", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_perf", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_mds_metadata", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_usage", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_usage_by_class", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_rook_v01", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_memory_metrics", 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_options_bluestore" 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "config": { 2024-09-14T03:37:22.563 INFO:tasks.workunit.client.0.smithi110.stdout: "active_changed": [ 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_debug", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_service_ticket_ttl", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "bdev_debug_aio", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "client_check_pool_perm", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "client_mount_gid", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "client_mount_uid", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "client_quota", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "container_image", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "daemonize", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mgr", 2024-09-14T03:37:22.564 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mon", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_ms", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_osd", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_paxos", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "keyring", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_block_size", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_cache_size", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_compression", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_log", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_write_buffer_size", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_file", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_journald", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_stderr", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_allow_pool_delete", 2024-09-14T03:37:22.565 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_clock_drift_allowed", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_file", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_journald", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_stderr", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_data_avail_warn", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_election_default_strategy", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_host", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_max_pg_per_osd", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:37:22.566 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_bug", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_old_message", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "no_config_file", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_default_list", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_load_list", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:37:22.567 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_op_order", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_shutdown", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_profile", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-14T03:37:22.568 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_lim", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_res", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target_autotune", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue_cut_off", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recover_clone_overlap", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recovery_max_chunk", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_load_threshold", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_max_interval", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:37:22.569 INFO:tasks.workunit.client.0.smithi110.stdout: "public_addr", 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "public_addrv", 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "public_network", 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd_default_features", 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd_qos_exclude_ops", 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "setgroup", 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "setuser" 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster_changed": [ 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:37:22.570 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_debug", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_service_ticket_ttl", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "bdev_debug_aio", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "container_image", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mgr", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mon", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_ms", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_osd", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_paxos", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_file", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_journald", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_stderr", 2024-09-14T03:37:22.571 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_join_fs", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/allow_ptrace", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/container_init", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/migration_current", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/use_repo_digest", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/ssl_server_port", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/orchestrator/orchestrator", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/channel_ident", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/channel_perf", 2024-09-14T03:37:22.572 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/description", 2024-09-14T03:37:22.573 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/enabled", 2024-09-14T03:37:22.573 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/organization", 2024-09-14T03:37:22.573 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_allow_pool_delete", 2024-09-14T03:37:22.573 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_clock_drift_allowed", 2024-09-14T03:37:22.573 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_file", 2024-09-14T03:37:22.573 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_data_avail_warn", 2024-09-14T03:37:22.573 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_election_default_strategy", 2024-09-14T03:37:22.573 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_max_pg_per_osd", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:37:22.574 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_bug", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_old_message", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_default_list", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_load_list", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_op_order", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_shutdown", 2024-09-14T03:37:22.575 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_profile", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target_autotune", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue_cut_off", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recover_clone_overlap", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recovery_max_chunk", 2024-09-14T03:37:22.576 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_load_threshold", 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_max_interval", 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "public_network", 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_cache_enabled", 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_enable_ops_log", 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_enable_usage_log" 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "contact": null, 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "crashes": [], 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "created": "2024-09-14T03:28:36.548032Z", 2024-09-14T03:37:22.577 INFO:tasks.workunit.client.0.smithi110.stdout: "crush": { 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_algs": { 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "straw2": 3 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_sizes": { 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "2": 1, 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "4": 2 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_types": { 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "1": 2, 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "11": 1 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "compat_weight_set": false, 2024-09-14T03:37:22.578 INFO:tasks.workunit.client.0.smithi110.stdout: "device_classes": [ 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: 8 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_buckets": 6, 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_devices": 8, 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_rules": 1, 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_types": 12, 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_weight_sets": 0, 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "tunables": { 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "allowed_bucket_algs": 54, 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_local_fallback_tries": 0, 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_local_tries": 0, 2024-09-14T03:37:22.579 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_total_tries": 50, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_descend_once": 1, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_stable": 1, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_vary_r": 1, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v2_rules": 0, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v3_rules": 0, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v4_buckets": 1, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v5_rules": 0, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "legacy_tunables": 0, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "minimum_required_version": "jewel", 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "optimal_tunables": 1, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "profile": "jewel", 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables": 1, 2024-09-14T03:37:22.580 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables2": 1, 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables3": 1, 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables5": 1, 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "straw_calc_version": 1 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "description": "upgrade test cluster", 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "fs": { 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "feature_flags": { 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "enable_multiple": true, 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "ever_enabled_multiple": true 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.581 INFO:tasks.workunit.client.0.smithi110.stdout: "filesystems": [ 2024-09-14T03:37:22.582 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.582 INFO:tasks.workunit.client.0.smithi110.stdout: "approx_ctime": "2024-09", 2024-09-14T03:37:22.582 INFO:tasks.workunit.client.0.smithi110.stdout: "balancer_enabled": false, 2024-09-14T03:37:22.582 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.582 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_caps": 0, 2024-09-14T03:37:22.582 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_dns": 10, 2024-09-14T03:37:22.582 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_inos": 13, 2024-09-14T03:37:22.582 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_subtrees": 2, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "ever_allowed_features": 0, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "explicitly_allowed_features": 0, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "files": 0, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "max_mds": 1, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_data_pools": 1, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_in": 1, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_mds": 1, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": 0, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_standby_replay": 0, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_up": 1, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "snaps": 0, 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: "standby_count_wanted": 0 2024-09-14T03:37:22.583 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "num_standby_mds": 1, 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "total_num_mds": 2 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_stats": { 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 20684800, 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 443232, 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 24576, 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 499712, 2024-09-14T03:37:22.584 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 440, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 1231032, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 20, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1251840, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 15112680, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 21184512 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 20135936, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 338936, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 819200, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 0, 2024-09-14T03:37:22.585 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 347, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 939608, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 12, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 720896, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 14695856, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 20135936 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 46825472, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 1021104, 2024-09-14T03:37:22.586 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 655360, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 2752512, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1776, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 10563968, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 22, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 3210752, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 28752848, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 49577984 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 37543936, 2024-09-14T03:37:22.587 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 1221176, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 9969664, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1201, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 7854824, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 22, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1498368, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 24348128, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 47513600 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.b030e701d31b8bd7b2036a36a31490ca893f182a": { 2024-09-14T03:37:22.588 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 36438016, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 757600, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 745472, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 10018816, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1017, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 6780936, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 21, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2185984, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 23346584, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 46456832 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.589 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.0": { 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 273694720, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 997336, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 32768, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1366, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 5186112, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2325504, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 262433256, 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 273727488 2024-09-14T03:37:22.590 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.1": { 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272310272, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 733880, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 303104, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1312, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 5957568, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1444096, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 261422216, 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 272613376 2024-09-14T03:37:22.591 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.2": { 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272285696, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 722544, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 606208, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1257, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 5222520, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1929728, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 261658392, 2024-09-14T03:37:22.592 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 272891904 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.3": { 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 271556608, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 731464, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 24576, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 73728, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1231, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4548960, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1813248, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 261685848, 2024-09-14T03:37:22.593 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 271630336 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.4": { 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272752640, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 824184, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 139264, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1231, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4627344, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2310912, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 262237688, 2024-09-14T03:37:22.594 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 272891904 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.5": { 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 275095552, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 772872, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 4849664, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 24576, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 729088, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1269, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4808752, 2024-09-14T03:37:22.595 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2349824, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 262289864, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 275824640 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.6": { 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272932864, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 1037552, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 1007616, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1248, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4617352, 2024-09-14T03:37:22.596 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2390784, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 262134664, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 273940480 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.7": { 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272392192, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 797848, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 32768, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 1335296, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1264, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:22.597 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 5006168, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1852160, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 261950736, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 273727488 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "hosts": { 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num": 2, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mds": 2, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mgr": 2, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mon": 2, 2024-09-14T03:37:22.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_osd": 2 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "io_rate": { 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_stats_delta": { 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 0, 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 0, 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "stamp_delta": "12.002394", 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:22.599 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:22.600 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:22.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 146, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 248, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 32, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 0, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:22.602 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 0, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 0 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "leaderboard": false, 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "license": "sharing-1-0", 2024-09-14T03:37:22.603 INFO:tasks.workunit.client.0.smithi110.stdout: "mempool": { 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.604 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.605 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.606 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.607 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.608 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.609 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 222183, 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 68 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 50488, 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 208 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.610 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 7944, 2024-09-14T03:37:22.611 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 135 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.613 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.614 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.615 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.616 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.617 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.618 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 149074, 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 33 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 856, 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 2 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.619 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.620 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.621 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.622 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.623 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.624 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.625 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.626 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.627 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1144378, 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 361 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 352, 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 4 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.629 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6592, 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 134 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12744, 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1546 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 10624, 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 206 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.631 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.632 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.633 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.634 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.635 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.636 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1312854, 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 403 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.637 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.638 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6592, 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 134 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12744, 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1546 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5312, 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 103 2024-09-14T03:37:22.639 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.b030e701d31b8bd7b2036a36a31490ca893f182a": { 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.640 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.641 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.642 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.643 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.644 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.645 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1094822, 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 348 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.647 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6592, 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 134 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.648 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12744, 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1546 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5312, 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 103 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.649 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.0": { 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:22.650 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20608, 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.651 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 112, 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8832, 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20608, 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.652 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5840, 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 73 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4096, 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 54642, 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9929 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 136000, 2024-09-14T03:37:22.653 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 200 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 11124, 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 713 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 31676, 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 200 2024-09-14T03:37:22.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 426628, 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 93 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.655 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2447296, 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 195 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5544, 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 63 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 521280, 2024-09-14T03:37:22.656 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 45 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 70304, 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 140 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.657 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234064, 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3472 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.658 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.1": { 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.659 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.660 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19040, 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 896, 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8160, 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.661 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19040, 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4640, 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 58 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12288, 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 48031, 2024-09-14T03:37:22.662 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8777 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 124440, 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 183 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 9040, 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 534 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.663 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 27056, 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 183 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 736, 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 420108, 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 92 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.664 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12096, 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2410957, 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 174 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4840, 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 55 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.665 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 428608, 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 37 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8528, 2024-09-14T03:37:22.666 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 22 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234064, 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3472 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.2": { 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.668 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19040, 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1120, 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 10 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8160, 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.670 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19040, 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4400, 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 55 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 26471, 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 6 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.671 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 46086, 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8540 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 121040, 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 178 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8368, 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 528 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.672 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 25776, 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 178 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 736, 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 358142, 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 77 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.673 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12096, 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2427224, 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 180 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.674 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4576, 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 52 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 359104, 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 31 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8320, 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 24 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234576, 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3496 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.676 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.677 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.3": { 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:22.678 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19600, 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 336, 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.679 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8400, 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19600, 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 3440, 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 43 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.680 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4096, 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 47348, 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8818 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 125120, 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 9060, 2024-09-14T03:37:22.681 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 545 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 26472, 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2208, 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.682 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 247335, 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 50 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 30637, 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2405411, 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 179 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.683 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4752, 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 54 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 417024, 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 36 2024-09-14T03:37:22.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 10400, 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 20 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234064, 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3472 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.685 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.686 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.4": { 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.687 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19600, 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 560, 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 5 2024-09-14T03:37:22.688 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8400, 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19600, 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 3440, 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 43 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.689 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 48760, 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9014 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 127840, 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 188 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 9364, 2024-09-14T03:37:22.690 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 546 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 27496, 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 188 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 736, 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:22.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 895536, 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 65 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12096, 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2397348, 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 168 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.692 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4752, 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 54 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 451776, 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 39 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.693 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8944, 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 18 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234320, 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3484 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.694 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.5": { 2024-09-14T03:37:22.695 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.696 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19376, 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 173 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 224, 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 2 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8304, 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 173 2024-09-14T03:37:22.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19376, 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 173 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2960, 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 37 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.698 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 49818, 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9103 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 129200, 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 190 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 9692, 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 547 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.699 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 28344, 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 190 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2208, 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.700 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 812795, 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 45 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 30637, 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2427458, 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 186 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5544, 2024-09-14T03:37:22.701 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 63 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 486528, 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 42 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.702 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 13520, 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 26 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234832, 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3508 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.703 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.6": { 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.704 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.705 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20384, 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8736, 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.706 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20384, 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2640, 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 33 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 49123, 2024-09-14T03:37:22.707 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9296 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 127160, 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 187 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 10604, 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 703 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 28604, 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 187 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 829053, 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 49 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.709 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2404108, 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 166 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4664, 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 53 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.710 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 451776, 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 39 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 69472, 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 144 2024-09-14T03:37:22.711 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234832, 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3508 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.712 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.7": { 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.713 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:22.714 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20384, 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1232, 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 11 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8736, 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20384, 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:22.715 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2240, 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 28 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 286720, 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 11 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 53184, 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9789 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.716 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 133960, 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 197 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 11388, 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 710 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:22.717 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 31164, 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 197 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 183505, 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 35 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.718 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2407186, 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 198 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6336, 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 72 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:22.719 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 532864, 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 46 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 73112, 2024-09-14T03:37:22.720 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234064, 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3472 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.721 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: "metadata": { 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:22.722 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 2 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 2 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:22.723 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 2 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 2 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:22.724 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 3 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 3 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:22.725 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 3 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 3 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.726 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 8 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 8 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:22.727 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 8 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 8 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_objectstore": { 2024-09-14T03:37:22.728 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": 8 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "rotational": { 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "0": 8 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "features": { 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "optional": [], 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "persistent": [ 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "kraken", 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "luminous", 2024-09-14T03:37:22.729 INFO:tasks.workunit.client.0.smithi110.stdout: "mimic", 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap-prune", 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "nautilus", 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "octopus", 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "pacific", 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "elector-pinging", 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "quincy" 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "ipv4_addr_mons": 6, 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "ipv6_addr_mons": 0, 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "min_mon_release": 17, 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "v1_addr_mons": 3, 2024-09-14T03:37:22.730 INFO:tasks.workunit.client.0.smithi110.stdout: "v2_addr_mons": 3 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "organization": "ceph-qa", 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster_network": false, 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "require_min_compat_client": "luminous", 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "require_osd_release": "quincy" 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_perf_histograms": [ 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: "axes": [ 2024-09-14T03:37:22.731 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Latency (usec)", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 100000, 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 99999]", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[100000, 199999]", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[200000, 399999]", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[400000, 799999]", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[800000, 1599999]", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[1600000, 3199999]", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[3200000, 6399999]", 2024-09-14T03:37:22.732 INFO:tasks.workunit.client.0.smithi110.stdout: "[6400000, 12799999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[12800000, 25599999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[25600000, 51199999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[51200000, 102399999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[102400000, 204799999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[204800000, 409599999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[409600000, 819199999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[819200000, 1638399999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[1638400000, 3276799999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[3276800000, 6553599999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[6553600000, 13107199999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[13107200000, 26214399999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[26214400000, 52428799999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[52428800000, 104857599999]", 2024-09-14T03:37:22.733 INFO:tasks.workunit.client.0.smithi110.stdout: "[104857600000, 209715199999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[209715200000, 419430399999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[419430400000, 838860799999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "[53687091200000, None]" 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.734 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Request size (bytes)", 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 512, 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 511]", 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "[512, 1023]", 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "[1024, 2047]", 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "[2048, 4095]", 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "[4096, 8191]", 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "[8192, 16383]", 2024-09-14T03:37:22.735 INFO:tasks.workunit.client.0.smithi110.stdout: "[16384, 32767]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[32768, 65535]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[65536, 131071]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[131072, 262143]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[262144, 524287]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[524288, 1048575]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[1048576, 2097151]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[2097152, 4194303]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[4194304, 8388607]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[8388608, 16777215]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[16777216, 33554431]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[33554432, 67108863]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[67108864, 134217727]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[134217728, 268435455]", 2024-09-14T03:37:22.736 INFO:tasks.workunit.client.0.smithi110.stdout: "[268435456, 536870911]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[536870912, 1073741823]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[1073741824, 2147483647]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[2147483648, 4294967295]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[4294967296, 8589934591]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[8589934592, 17179869183]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[17179869184, 34359738367]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[34359738368, 68719476735]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[68719476736, 137438953471]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[137438953472, 274877906943]", 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "[274877906944, None]" 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.737 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "osds": [ 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 0, 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.738 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.739 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.740 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 1, 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.741 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.742 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.743 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 2, 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.744 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.745 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 3, 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.746 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.747 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.748 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 4, 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.749 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.750 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.751 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 5, 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.752 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.753 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.754 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 6, 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.755 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.756 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.757 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 7, 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.758 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 88, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.759 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.760 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: "axes": [ 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:22.761 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Latency (usec)", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 100000, 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 99999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[100000, 199999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[200000, 399999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[400000, 799999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[800000, 1599999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[1600000, 3199999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[3200000, 6399999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[6400000, 12799999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[12800000, 25599999]", 2024-09-14T03:37:22.762 INFO:tasks.workunit.client.0.smithi110.stdout: "[25600000, 51199999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[51200000, 102399999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[102400000, 204799999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[204800000, 409599999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[409600000, 819199999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[819200000, 1638399999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[1638400000, 3276799999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[3276800000, 6553599999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[6553600000, 13107199999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[13107200000, 26214399999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[26214400000, 52428799999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[52428800000, 104857599999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[104857600000, 209715199999]", 2024-09-14T03:37:22.763 INFO:tasks.workunit.client.0.smithi110.stdout: "[209715200000, 419430399999]", 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "[419430400000, 838860799999]", 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "[53687091200000, None]" 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:22.764 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Request size (bytes)", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 512, 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 511]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[512, 1023]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[1024, 2047]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[2048, 4095]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[4096, 8191]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[8192, 16383]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[16384, 32767]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[32768, 65535]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[65536, 131071]", 2024-09-14T03:37:22.765 INFO:tasks.workunit.client.0.smithi110.stdout: "[131072, 262143]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[262144, 524287]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[524288, 1048575]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[1048576, 2097151]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[2097152, 4194303]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[4194304, 8388607]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[8388608, 16777215]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[16777216, 33554431]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[33554432, 67108863]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[67108864, 134217727]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[134217728, 268435455]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[268435456, 536870911]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[536870912, 1073741823]", 2024-09-14T03:37:22.766 INFO:tasks.workunit.client.0.smithi110.stdout: "[1073741824, 2147483647]", 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "[2147483648, 4294967295]", 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "[4294967296, 8589934591]", 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "[8589934592, 17179869183]", 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "[17179869184, 34359738367]", 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "[34359738368, 68719476735]", 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "[68719476736, 137438953471]", 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "[137438953472, 274877906943]", 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "[274877906944, None]" 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: "osds": [ 2024-09-14T03:37:22.767 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 0, 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.768 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.769 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 1, 2024-09-14T03:37:22.770 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.771 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.772 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 2, 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.773 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.774 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.775 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 3, 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.776 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.777 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.778 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 4, 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.779 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.780 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.781 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 5, 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.782 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.783 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.784 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 6, 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.785 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.786 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 7, 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.787 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.788 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.789 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "axes": [ 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:22.790 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Latency (usec)", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 100000, 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 99999]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[100000, 199999]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[200000, 399999]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[400000, 799999]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[800000, 1599999]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[1600000, 3199999]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[3200000, 6399999]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[6400000, 12799999]", 2024-09-14T03:37:22.791 INFO:tasks.workunit.client.0.smithi110.stdout: "[12800000, 25599999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[25600000, 51199999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[51200000, 102399999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[102400000, 204799999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[204800000, 409599999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[409600000, 819199999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[819200000, 1638399999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[1638400000, 3276799999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[3276800000, 6553599999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[6553600000, 13107199999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[13107200000, 26214399999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[26214400000, 52428799999]", 2024-09-14T03:37:22.792 INFO:tasks.workunit.client.0.smithi110.stdout: "[52428800000, 104857599999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[104857600000, 209715199999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[209715200000, 419430399999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[419430400000, 838860799999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "[53687091200000, None]" 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:22.793 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Request size (bytes)", 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 512, 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 511]", 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "[512, 1023]", 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "[1024, 2047]", 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "[2048, 4095]", 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "[4096, 8191]", 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "[8192, 16383]", 2024-09-14T03:37:22.794 INFO:tasks.workunit.client.0.smithi110.stdout: "[16384, 32767]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[32768, 65535]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[65536, 131071]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[131072, 262143]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[262144, 524287]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[524288, 1048575]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[1048576, 2097151]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[2097152, 4194303]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[4194304, 8388607]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[8388608, 16777215]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[16777216, 33554431]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[33554432, 67108863]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[67108864, 134217727]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[134217728, 268435455]", 2024-09-14T03:37:22.795 INFO:tasks.workunit.client.0.smithi110.stdout: "[268435456, 536870911]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[536870912, 1073741823]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[1073741824, 2147483647]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[2147483648, 4294967295]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[4294967296, 8589934591]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[8589934592, 17179869183]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[17179869184, 34359738367]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[34359738368, 68719476735]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[68719476736, 137438953471]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[137438953472, 274877906943]", 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "[274877906944, None]" 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:22.796 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "osds": [ 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 0, 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.797 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.798 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 1, 2024-09-14T03:37:22.799 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.800 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.801 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 2, 2024-09-14T03:37:22.802 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.803 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.804 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 3, 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.805 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.806 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.807 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 4, 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.808 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.809 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.810 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 5, 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.811 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.812 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.813 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 6, 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.814 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.815 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 7, 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.816 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.817 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.818 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: "axes": [ 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:22.819 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Latency (usec)", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 100000, 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 99999]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[100000, 199999]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[200000, 399999]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[400000, 799999]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[800000, 1599999]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[1600000, 3199999]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[3200000, 6399999]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[6400000, 12799999]", 2024-09-14T03:37:22.820 INFO:tasks.workunit.client.0.smithi110.stdout: "[12800000, 25599999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[25600000, 51199999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[51200000, 102399999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[102400000, 204799999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[204800000, 409599999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[409600000, 819199999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[819200000, 1638399999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[1638400000, 3276799999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[3276800000, 6553599999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[6553600000, 13107199999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[13107200000, 26214399999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[26214400000, 52428799999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[52428800000, 104857599999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[104857600000, 209715199999]", 2024-09-14T03:37:22.821 INFO:tasks.workunit.client.0.smithi110.stdout: "[209715200000, 419430399999]", 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "[419430400000, 838860799999]", 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "[53687091200000, None]" 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:22.822 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Request size (bytes)", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 512, 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 511]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[512, 1023]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[1024, 2047]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[2048, 4095]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[4096, 8191]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[8192, 16383]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[16384, 32767]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[32768, 65535]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[65536, 131071]", 2024-09-14T03:37:22.823 INFO:tasks.workunit.client.0.smithi110.stdout: "[131072, 262143]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[262144, 524287]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[524288, 1048575]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[1048576, 2097151]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[2097152, 4194303]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[4194304, 8388607]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[8388608, 16777215]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[16777216, 33554431]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[33554432, 67108863]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[67108864, 134217727]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[134217728, 268435455]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[268435456, 536870911]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[536870912, 1073741823]", 2024-09-14T03:37:22.824 INFO:tasks.workunit.client.0.smithi110.stdout: "[1073741824, 2147483647]", 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "[2147483648, 4294967295]", 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "[4294967296, 8589934591]", 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "[8589934592, 17179869183]", 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "[17179869184, 34359738367]", 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "[34359738368, 68719476735]", 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "[68719476736, 137438953471]", 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "[137438953472, 274877906943]", 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "[274877906944, None]" 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: "osds": [ 2024-09-14T03:37:22.825 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 0, 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.826 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.827 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 1, 2024-09-14T03:37:22.828 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 6, 3, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 13, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.829 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.830 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 2, 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.831 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.832 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.833 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 3, 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.834 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.835 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.836 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 4, 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.837 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.838 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.839 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 5, 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.840 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.841 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 6, 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.842 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.843 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.844 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 7, 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.845 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.846 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:22.847 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_counters": { 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: "caps": { 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.848 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_flush_ack": { 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_grant": { 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_revoke": { 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_trunc": { 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "dir_commit": { 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "dir_fetch": { 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "dir_merge": { 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "dir_split": { 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.850 INFO:tasks.workunit.client.0.smithi110.stdout: "exported_inodes": { 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "forward": { 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_cap_release": { 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_caps": { 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_caps_dirty": { 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_inode_file_caps": { 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "imported_inodes": { 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "inodes": { 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "inodes_expired": { 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "inodes_pinned": { 2024-09-14T03:37:22.852 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: "inodes_with_caps": { 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: "load_cent": { 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: "openino_dir_fetch": { 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: "process_request_cap_release": { 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "reply_latency": { 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 150712 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "request": { 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "root_rbytes": { 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "root_rfiles": { 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.854 INFO:tasks.workunit.client.0.smithi110.stdout: "root_rsnaps": { 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_reply": { 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "subtrees": { 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_cache": { 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_enqueue_scrub": { 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.855 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_exportdir": { 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_flush": { 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_fragmentdir": { 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_fragstats": { 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_inodestats": { 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_recovering_enqueued": { 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_recovering_prioritized": { 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_recovering_processing": { 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_strays": { 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_strays_delayed": { 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_strays_enqueuing": { 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_completed": { 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_started": { 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.858 INFO:tasks.workunit.client.0.smithi110.stdout: "strays_created": { 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: "strays_enqueued": { 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: "strays_migrated": { 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: "strays_reintegrated": { 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.859 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_log": { 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "ev": { 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "evadd": { 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "evex": { 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "evexd": { 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "evexg": { 2024-09-14T03:37:22.860 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "evtrm": { 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "jlat": { 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11187690 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "replayed": { 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "seg": { 2024-09-14T03:37:22.861 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "segadd": { 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "segex": { 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "segexd": { 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "segexg": { 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.862 INFO:tasks.workunit.client.0.smithi110.stdout: "segtrm": { 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_mem": { 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "cap": { 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "cap+": { 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "cap-": { 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.863 INFO:tasks.workunit.client.0.smithi110.stdout: "dir": { 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "dir+": { 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "dir-": { 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "dn": { 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "dn+": { 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:22.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "dn-": { 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "heap": { 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 207068 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "ino": { 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "ino+": { 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "ino-": { 2024-09-14T03:37:22.865 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: "rss": { 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 37880 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_server": { 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: "cap_acquisition_throttle": { 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: "cap_revoke_eviction": { 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.866 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_request": { 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_session": { 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 16 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_peer_request": { 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: "req_create_latency": { 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "req_getattr_latency": { 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 150712 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "req_getfilelock_latency": { 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "req_getvxattr_latency": { 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "req_link_latency": { 2024-09-14T03:37:22.868 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookup_latency": { 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookuphash_latency": { 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookupino_latency": { 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookupname_latency": { 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookupparent_latency": { 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookupsnap_latency": { 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.870 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lssnap_latency": { 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "req_mkdir_latency": { 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "req_mknod_latency": { 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "req_mksnap_latency": { 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "req_open_latency": { 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "req_readdir_latency": { 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "req_rename_latency": { 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.872 INFO:tasks.workunit.client.0.smithi110.stdout: "req_renamesnap_latency": { 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "req_rmdir_latency": { 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "req_rmsnap_latency": { 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "req_rmxattr_latency": { 2024-09-14T03:37:22.873 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setattr_latency": { 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setdirlayout_latency": { 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setfilelock_latency": { 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setlayout_latency": { 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setxattr_latency": { 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "req_symlink_latency": { 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.875 INFO:tasks.workunit.client.0.smithi110.stdout: "req_unlink_latency": { 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_sessions": { 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "average_load": { 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "avg_session_uptime": { 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 87 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "mdthresh_evicted": { 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "session_add": { 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "session_count": { 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "session_remove": { 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "sessions_open": { 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.877 INFO:tasks.workunit.client.0.smithi110.stdout: "sessions_stale": { 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: "total_load": { 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.878 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "purge_queue": { 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executed": { 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executing": { 2024-09-14T03:37:22.880 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executing_high_water": { 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executing_ops": { 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executing_ops_high_water": { 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_item_in_journal": { 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.881 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "election_call": { 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "election_lose": { 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "election_win": { 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_elections": { 2024-09-14T03:37:22.882 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": { 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: "session_add": { 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: "session_rm": { 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 185 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: "session_trim": { 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "paxos": { 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "accept_timeout": { 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "begin": { 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 933 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_bytes": { 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5015259 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_keys": { 2024-09-14T03:37:22.884 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2799 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_latency": { 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 135997710 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "collect": { 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_bytes": { 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.885 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_keys": { 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_latency": { 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_timeout": { 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_uncommitted": { 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.886 INFO:tasks.workunit.client.0.smithi110.stdout: "commit": { 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 933 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_bytes": { 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4948141 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_keys": { 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5347 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_latency": { 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 125875915 2024-09-14T03:37:22.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_ack_timeout": { 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_timeout": { 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn": { 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn_latency": { 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:22.888 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh": { 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 936 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh_latency": { 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 936, 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2195963021 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "restart": { 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state": { 2024-09-14T03:37:22.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_bytes": { 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 473 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_keys": { 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "start_leader": { 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "start_peon": { 2024-09-14T03:37:22.890 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state": { 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_bytes": { 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_keys": { 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_latency": { 2024-09-14T03:37:22.891 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache": { 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1020054731 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 46956544 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 42328064 2024-09-14T03:37:22.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2147483648 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4628480 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:full": { 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277021587 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 273125 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.895 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 70832520 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:inc": { 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:22.896 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277021587 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 39004 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71066641 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.899 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:kv": { 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 322961408 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 184681058 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.901 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71171486 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb": { 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: "compact": { 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_completed": { 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lasted": { 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 31253947 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.903 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_len": { 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_merge": { 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_running": { 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "get_latency": { 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 41572, 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 149350771 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:37:22.904 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.905 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_latency": { 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_sync_latency": { 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2030, 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 264315066 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "election_call": { 2024-09-14T03:37:22.906 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: "election_lose": { 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: "election_win": { 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_elections": { 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": { 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9 2024-09-14T03:37:22.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "session_add": { 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "session_rm": { 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 50 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "session_trim": { 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "paxos": { 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "accept_timeout": { 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "begin": { 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 812 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_bytes": { 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 812, 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3974355 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_keys": { 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.909 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_latency": { 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 812, 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 107997942 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "collect": { 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_bytes": { 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 219 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_keys": { 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:22.910 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_latency": { 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_timeout": { 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_uncommitted": { 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "commit": { 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 813 2024-09-14T03:37:22.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_bytes": { 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_keys": { 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_latency": { 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.912 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_ack_timeout": { 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_timeout": { 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn": { 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn_latency": { 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh": { 2024-09-14T03:37:22.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 813 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh_latency": { 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 813, 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1637968651 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "restart": { 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state": { 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_bytes": { 2024-09-14T03:37:22.914 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_keys": { 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "start_leader": { 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "start_peon": { 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state": { 2024-09-14T03:37:22.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 813 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_bytes": { 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 813, 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7796231 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_keys": { 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 813, 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5498 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_latency": { 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 813, 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 118997720 2024-09-14T03:37:22.916 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache": { 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1020054731 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44892160 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 34603008 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2147483648 2024-09-14T03:37:22.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10289152 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:full": { 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277020525 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 273555 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 70833152 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:inc": { 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.921 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277020525 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 41407 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71065300 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:kv": { 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.924 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 322961408 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 184680350 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.925 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71172194 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.927 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb": { 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "compact": { 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_completed": { 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lasted": { 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 20276919 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_len": { 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.928 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_merge": { 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_running": { 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "get_latency": { 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 35085, 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 79047830 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.929 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.930 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_latency": { 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_sync_latency": { 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1793, 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 255983165 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.b030e701d31b8bd7b2036a36a31490ca893f182a": { 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "election_call": { 2024-09-14T03:37:22.931 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "election_lose": { 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "election_win": { 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_elections": { 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": { 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.932 INFO:tasks.workunit.client.0.smithi110.stdout: "session_add": { 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: "session_rm": { 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 78 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: "session_trim": { 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: "paxos": { 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: "accept_timeout": { 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.933 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "begin": { 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 787 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_bytes": { 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3899829 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_keys": { 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_latency": { 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:22.934 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 86998539 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "collect": { 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_bytes": { 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_keys": { 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_latency": { 2024-09-14T03:37:22.935 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 999984 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_timeout": { 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_uncommitted": { 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "commit": { 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 787 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_bytes": { 2024-09-14T03:37:22.936 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_keys": { 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_latency": { 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_ack_timeout": { 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_timeout": { 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn": { 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn_latency": { 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh": { 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 787 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.938 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh_latency": { 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1769970170 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "restart": { 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state": { 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_bytes": { 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_keys": { 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "start_leader": { 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "start_peon": { 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state": { 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 787 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.940 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_bytes": { 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7650883 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_keys": { 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5312 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_latency": { 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 90998468 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.941 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache": { 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1020054731 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 43835392 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 32112640 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2147483648 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:22.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11722752 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:full": { 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.943 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277020525 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 273555 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.944 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.945 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 70833152 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:inc": { 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:22.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277020525 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 41407 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.947 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.948 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.949 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71065300 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:kv": { 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 322961408 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.950 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 184680350 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.951 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.952 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71172194 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.953 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb": { 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "compact": { 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_completed": { 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lasted": { 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 30515755 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_len": { 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.954 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_merge": { 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_running": { 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "get_latency": { 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 33598, 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 80203642 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.956 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_latency": { 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_sync_latency": { 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1739, 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 202908699 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.0": { 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:22.957 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2420736 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1630208 2024-09-14T03:37:22.960 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1490944 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:22.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.963 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:22.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.965 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:22.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3985408 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1306624 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:22.967 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270560 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.968 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 19, 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56215 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 347, 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 59208500 2024-09-14T03:37:22.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 337, 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10853986 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 347, 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 375197 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 347, 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 59583697 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:22.970 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 30231 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10095 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11661 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 721 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:22.972 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 224 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 115, 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1478999 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 276, 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 248347 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:22.973 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 161, 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1006797 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 49262 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5356489 2024-09-14T03:37:22.974 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 101, 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6737607 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 101, 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 494004958405 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 101, 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 135868615904 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:22.975 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 57190608630 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 51155 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 405567 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:22.976 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 122807626 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 139827 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 61621572 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 30093122 2024-09-14T03:37:22.977 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 903228 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 61, 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 274973 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 220343525 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:22.978 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 35818606 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2880808 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861437685 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:22.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270974976 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270721024 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137345945 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 253952 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:22.980 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85496386 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.982 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73883070 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.984 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641222901 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.985 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71806651 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.987 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56997591 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68831065 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.990 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641222901 2024-09-14T03:37:22.991 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9926 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.992 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 273675 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:22.993 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71525178 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.994 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.995 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 604 2024-09-14T03:37:22.996 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 6, 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44652978 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 7, 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3427038 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:22.997 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 6, 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 43098701 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.998 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:22.999 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.000 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6 2024-09-14T03:37:23.001 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 604 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 6, 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44652978 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 7, 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3427038 2024-09-14T03:37:23.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 6, 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 43098701 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.003 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305795072 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.1": { 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.004 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:23.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2064384 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:23.006 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1413120 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:23.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.008 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:23.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:23.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3411968 2024-09-14T03:37:23.013 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 696320 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 210139 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:23.014 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.015 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 26, 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 79973 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56417516 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:23.016 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 282, 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10370644 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 386709 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56804225 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.017 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 31535 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9184 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.018 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18983 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 660 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 227 2024-09-14T03:37:23.019 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 116, 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1991654 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 272, 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 246039 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 156, 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1318838 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.020 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 106809 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5502311 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 96, 2024-09-14T03:37:23.021 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6718397 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 95, 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 346766756657 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 96, 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 105748426944 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 348, 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 27823266923 2024-09-14T03:37:23.022 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 348, 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 36602 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 351053 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 198969530 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.023 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 109650 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 59104874 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18677552 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 311378 2024-09-14T03:37:23.024 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 55, 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 318006 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 282653249 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 22579974 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.025 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2250577 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861437685 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269860864 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:23.026 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269443072 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137345945 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 417792 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.027 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85497569 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.029 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73873695 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.030 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641231768 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71797784 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998379 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.036 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68830277 2024-09-14T03:37:23.037 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.038 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641231768 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 252908 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.040 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71547004 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.041 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 37 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 97 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1640827 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:23.043 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 97, 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 242941886 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 608 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 97, 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 33927786 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 97, 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 120526059 2024-09-14T03:37:23.044 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 18, 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2595833 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 608 2024-09-14T03:37:23.045 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 18, 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2313159 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 18, 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1888538 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.046 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9485040 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.047 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4654053 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8901335 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:23.048 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1640827 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 67, 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 230861013 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 67, 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 26960574 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.049 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 67, 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 109736186 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 590368 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:23.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305188864 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.2": { 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:23.051 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1769472 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.053 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1200128 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1060864 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.054 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:23.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.056 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.057 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.058 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:23.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2904064 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 696320 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:23.061 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 239697 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.062 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 19696 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:23.063 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 242, 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 38437610 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 239, 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7391541 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 242, 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 365978 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:23.064 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 242, 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 38803588 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 36254 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:23.065 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1680 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5672 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.066 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 513 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 208 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 96, 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1489412 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 227, 2024-09-14T03:37:23.067 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 225297 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 131, 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 992963 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85488 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.068 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5326499 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 90, 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5724775 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 89, 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 313317598528 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:23.069 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 90, 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 130779151433 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 289, 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 64949844443 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 289, 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 29490 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 335042 2024-09-14T03:37:23.070 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67983316 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95019 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 30687428 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.071 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 16404673 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312144 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 50, 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 262564 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 120777644 2024-09-14T03:37:23.072 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 19715667 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2311227 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861437685 2024-09-14T03:37:23.073 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270139392 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269410304 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137345945 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 729088 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.074 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85497502 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.075 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.076 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 20480 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.077 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73865570 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.078 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641231265 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.079 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.080 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71798287 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:23.081 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998334 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.083 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.083 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.083 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.083 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.084 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.084 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.084 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.084 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.084 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.084 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.085 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68830322 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.086 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641231265 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.087 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 245836 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.088 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71554579 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.089 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:23.090 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 31 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.091 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 108307392 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4455016 2024-09-14T03:37:23.092 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 28163644 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:23.093 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:23.094 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:23.095 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.096 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 108307392 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:23.097 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4455016 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 28163644 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:23.098 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305188864 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.3": { 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:23.099 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.100 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1142784 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.101 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:23.102 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 897024 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 757760 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.103 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:23.104 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:23.105 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.106 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:23.107 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:23.108 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1974272 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.109 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 716800 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.110 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6719 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.111 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 168, 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 29425379 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 156, 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11524890 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 168, 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 211626 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.112 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 168, 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 29637005 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 36962 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:23.113 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 833 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.114 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8126 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 316 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 222 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 84, 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 763951 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:23.115 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 140, 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 287256 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 56, 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 216863 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.116 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4073079 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 45, 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2984112 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 42, 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 326857821236 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.117 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 45, 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 66141850537 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 230, 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 509494 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 230, 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23501 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.118 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 336837 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 91660724 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65953 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 43687634 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.119 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11172381 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 314872 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 36, 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 190044 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:23.120 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 150958930 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13643909 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1639711 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:23.121 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861437685 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 268877824 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 268681216 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137345945 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.122 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 196608 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.123 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85498046 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.124 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.125 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73881410 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.126 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641235350 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.127 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.128 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71794202 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.129 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998697 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.130 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.131 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.132 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68829959 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.133 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641235350 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.134 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6954 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 248981 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.135 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71540395 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:23.136 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:23.137 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 36 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.138 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 720598 2024-09-14T03:37:23.139 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.140 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:23.141 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:23.142 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.143 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.144 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 720598 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.145 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305205248 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.4": { 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:23.146 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.147 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1302528 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.148 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1003520 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.149 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 864256 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:23.150 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.151 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:23.152 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.153 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:23.154 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.155 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2240512 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 716800 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 341939 2024-09-14T03:37:23.156 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.157 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 11, 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 32115 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 194, 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45336484 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 186, 2024-09-14T03:37:23.158 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6545617 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 194, 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 220758 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 194, 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45557242 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.159 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 34110 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6665 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:23.160 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9762 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 371 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 235 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:23.161 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 92, 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1179161 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 153, 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 214272 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 61, 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 533475 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.162 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 104651 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4464750 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 41, 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2586832 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:23.163 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 40, 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 205514579974 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 41, 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 61296360286 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 263, 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8307274851 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:23.164 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 263, 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 46187 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 241652 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 101423670 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.165 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 150543 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 58578829 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12659084 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312859 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:23.166 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 39, 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 168919 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 177403608 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15337113 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:23.167 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2149014 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861443778 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270139392 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269836288 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.168 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137353113 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 303104 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.169 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85498317 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.170 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.171 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73885235 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.172 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641237377 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.173 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.174 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.175 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71792175 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.176 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998878 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.177 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.178 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68829778 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.179 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641237377 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.180 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 256053 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.181 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71538250 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:23.182 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:23.183 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 39 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.184 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 22 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7100463 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.185 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 618950 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7026259 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.186 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.187 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:23.188 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.189 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 22 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7100463 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:23.190 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 618950 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7026259 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.191 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305209344 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.192 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.5": { 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.193 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1101824 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.194 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.195 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 811008 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 671744 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:23.196 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.197 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:23.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.199 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.201 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1847296 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 708608 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.202 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 137847 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.203 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18987 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 147, 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45446094 2024-09-14T03:37:23.204 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 145, 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3995658 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 147, 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 724148 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 147, 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 46170242 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.205 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 32122 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4503 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.206 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4066 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 335 2024-09-14T03:37:23.207 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 242 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 86, 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 818237 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 135, 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 264641 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:23.208 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 49, 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 350694 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 77891 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2788885 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.209 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 40, 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2849106 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 37, 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270364427588 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 40, 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 58289044221 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:23.210 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 267, 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8140157513 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 267, 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 27585 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 222640 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.211 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 111535458 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 96261 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 145447605 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12680853 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.212 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 314318 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 32, 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 149299 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 272732474 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.213 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14697154 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1968038 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861443778 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270974976 2024-09-14T03:37:23.214 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270098432 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137353113 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 876544 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:23.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85498154 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.216 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73885398 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.218 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641236156 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.219 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.220 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.221 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71793396 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.222 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998769 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.223 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.224 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68829887 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.225 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641236156 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.226 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6956 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 257377 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.227 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71531191 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:23.228 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:23.229 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 42 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.231 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:23.232 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.233 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.234 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.235 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.236 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.237 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 590368 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.238 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305197056 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.6": { 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:23.239 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:23.240 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1220608 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:23.241 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 942080 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 802816 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.242 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:23.243 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.244 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:23.245 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.246 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.247 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2097152 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.248 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1298432 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.249 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 27563 2024-09-14T03:37:23.250 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 179, 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 40751332 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 168, 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9034851 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:23.251 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 179, 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 156284 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 179, 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 40907616 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.252 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 39098 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4627 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.253 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10553 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 347 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 244 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 92, 2024-09-14T03:37:23.254 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 832669 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 145, 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 215531 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 53, 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 392999 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.255 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4888241 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 35, 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2554457 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:23.256 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 35, 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 440087844585 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 35, 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 48409185137 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 439957 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 28020 2024-09-14T03:37:23.257 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 828161 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 98400919 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 80010 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.258 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 60646315 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11572823 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 904821 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 28, 2024-09-14T03:37:23.259 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 122883 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 176373022 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 16305052 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2159421 2024-09-14T03:37:23.260 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861443778 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 271187968 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270016512 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137353113 2024-09-14T03:37:23.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1171456 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.262 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85498092 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.263 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.264 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73885460 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.265 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641235693 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.266 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.267 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71793859 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.268 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998728 2024-09-14T03:37:23.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.270 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68829928 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.272 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641235693 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7095 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 258267 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.274 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71530625 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:23.276 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 39 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2804 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 13, 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68458525 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 15, 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6971829 2024-09-14T03:37:23.278 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 13, 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 38597408 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:23.279 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.280 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.281 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.282 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2804 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 13, 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68458525 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.283 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 15, 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6971829 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 13, 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 38597408 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:23.284 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 590368 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305786880 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.7": { 2024-09-14T03:37:23.285 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.286 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:23.287 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.287 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.287 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:23.287 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.287 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.287 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:23.287 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:23.287 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.288 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:23.288 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1228800 2024-09-14T03:37:23.288 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.288 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:23.288 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.289 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 958464 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 819200 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:23.290 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:23.291 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.292 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:23.293 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.294 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:23.295 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2121728 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.296 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1298432 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 137018 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.297 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 29349 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:23.298 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 183, 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 52582473 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 170, 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13584141 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 183, 2024-09-14T03:37:23.299 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 133383 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 183, 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 52715856 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44957 2024-09-14T03:37:23.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1727 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.301 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9046 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 549 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 268 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 114, 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1218152 2024-09-14T03:37:23.302 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 181, 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 213615 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 67, 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 600635 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:23.303 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 17356 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3714527 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 24, 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1670581 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 24, 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 274136803677 2024-09-14T03:37:23.304 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 24, 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 34902155110 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 436083 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 41083 2024-09-14T03:37:23.305 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1418046 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 179425563 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 101678 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.306 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 120748548 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11964813 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 903104 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:23.307 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 23, 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 97298 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 317328000 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14767434 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1973643 2024-09-14T03:37:23.308 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861443778 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270974976 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269631488 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:23.309 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137353113 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1343488 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.310 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85488841 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.311 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 139264 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.312 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73755447 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.313 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641166308 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.314 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.315 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71863244 2024-09-14T03:37:23.316 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.317 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56992560 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.318 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.319 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68836096 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.320 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641166308 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.321 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 274596 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:23.322 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71590776 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:23.323 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:23.324 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 46 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 87 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.325 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 87, 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71925275 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 264333 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.326 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 89, 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 19822920 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 87, 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 50673872 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 60 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 60, 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8251032 2024-09-14T03:37:23.327 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 264333 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 60, 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7307006 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 60, 2024-09-14T03:37:23.328 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5852018 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 22 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 22, 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 19621401 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:23.329 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 22, 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8596498 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:23.330 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 22, 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18279099 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44052842 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.331 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 7, 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3919416 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 26542755 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:23.332 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 590368 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305786880 2024-09-14T03:37:23.333 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: "pools": [ 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr" 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:23.334 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 1, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 1, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 1, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115901194, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 1781760, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:23.335 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 1781760, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 2, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 2.450442480039783e-06, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 272, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 490496, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 590368, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 590368, 2024-09-14T03:37:23.336 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 1771104, 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 145, 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 1667072 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.337 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 32, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 32, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 2, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115901194, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 98304, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:23.338 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 98304, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 22, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 3410, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 3410, 2024-09-14T03:37:23.339 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 10230, 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 51, 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 19456 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.340 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 64, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 64, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 3, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115901194, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 0, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:23.341 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 0.0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 0, 2024-09-14T03:37:23.342 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 0, 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 0, 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 0 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd" 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.343 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 8, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 8, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 4, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115901194, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 12288, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:23.344 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 12288, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 2, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 1.689964435058755e-08, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 19, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 19, 2024-09-14T03:37:23.345 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 57, 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 2, 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 2048 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd": { 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: "mirroring_by_pool": [ 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: false 2024-09-14T03:37:23.346 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_images_by_pool": [ 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pools": 1 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "report_id": "2dc8c2a9-6f51-4b3c-b455-44a7525f06ca", 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "report_timestamp": "2024-09-14T03:37:22.447100", 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "report_version": 1, 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_stats": { 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "6.15.5" 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "rook": { 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster": { 2024-09-14T03:37:23.347 INFO:tasks.workunit.client.0.smithi110.stdout: "external-mode": null, 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "allow-multiple-per-node": null, 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "count": null, 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "max-id": null, 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "pvc": { 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "enabled": null 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "stretch": { 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "enabled": null 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "network": { 2024-09-14T03:37:23.348 INFO:tasks.workunit.client.0.smithi110.stdout: "provider": null 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: "storage": { 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: "device-set": { 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: "non-portable": null, 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: "portable": null, 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: "total": null 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: "csi": { 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:23.349 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "kubernetes": { 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "node": { 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "kubernetes-total": null, 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "with-ceph-daemons": null, 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-cephfs-plugin": null, 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-nfs-plugin": null, 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-rbd-plugin": null 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "usage": { 2024-09-14T03:37:23.350 INFO:tasks.workunit.client.0.smithi110.stdout: "storage-class": { 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket": null, 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs": null, 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "nfs": null, 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd": null, 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "total": null 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "services": {}, 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_per_pg": [ 2024-09-14T03:37:23.351 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.352 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.518530+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.518530+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.519367+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.353 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.354 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3f", 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:02:36.205002+0000", 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.355 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.357 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.358 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.359 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.360 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.514951+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.514951+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515212+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.361 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.362 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3e", 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:08:30.742131+0000", 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.363 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.364 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.365 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.366 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.367 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.504550+0000", 2024-09-14T03:37:23.368 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.504550+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.504777+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.369 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.370 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3d", 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:17:59.864492+0000", 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.372 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.373 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.374 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.375 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.521149+0000", 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.521149+0000", 2024-09-14T03:37:23.376 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521444+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:23.377 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.378 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3c", 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:27:10.004625+0000", 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.379 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.380 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.381 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.382 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.383 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512580+0000", 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512580+0000", 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513133+0000", 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:23.384 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.385 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3b", 2024-09-14T03:37:23.386 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:43:03.077106+0000", 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.387 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.388 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.389 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.390 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.391 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.507921+0000", 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.507921+0000", 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.508230+0000", 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.392 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.393 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3a", 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.394 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:59:37.418870+0000", 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.395 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.396 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.397 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.398 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.399 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.515022+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.515022+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515319+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:23.400 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.401 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.39", 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:51:51.089278+0000", 2024-09-14T03:37:23.402 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.403 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.404 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.405 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.406 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.407 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.522063+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.522063+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.522349+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:23.408 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.409 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.38", 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:16:15.378220+0000", 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.410 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.411 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.412 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.413 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.414 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511364+0000", 2024-09-14T03:37:23.415 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511364+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512031+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.416 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.417 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.37", 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:54:15.711934+0000", 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.418 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.419 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.420 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.421 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.422 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.518575+0000", 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.518575+0000", 2024-09-14T03:37:23.423 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.518932+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:23.424 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.425 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.36", 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:01:11.929183+0000", 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.426 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.427 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.428 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.429 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.430 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.517385+0000", 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.517385+0000", 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517977+0000", 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:23.431 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.432 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.433 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.35", 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:42:50.900533+0000", 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.434 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.435 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.436 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.438 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.439 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.510519+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.510519+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.510949+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.440 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.441 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.34", 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:06:16.860908+0000", 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.442 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.443 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.444 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.446 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.447 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512201+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512201+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512463+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.448 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.449 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.33", 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:04:46.716482+0000", 2024-09-14T03:37:23.450 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.451 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.452 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.454 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.455 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512773+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512773+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513215+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:23.456 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.457 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.32", 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:27:04.454480+0000", 2024-09-14T03:37:23.458 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.459 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.462 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.463 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.503212+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.503212+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.503785+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.464 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.465 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.31", 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:51:15.408242+0000", 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.466 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.467 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.470 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.953477+0000", 2024-09-14T03:37:23.471 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.953477+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.953866+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.472 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.473 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.30", 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 31, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:56:22.262447+0000", 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.474 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.475 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.477 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.478 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.506907+0000", 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.506907+0000", 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.507186+0000", 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.479 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.480 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2f", 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.481 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:10:47.570717+0000", 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.482 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.485 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.486 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.518795+0000", 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.518795+0000", 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.519104+0000", 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.487 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.488 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2e", 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.489 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:34:49.356603+0000", 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.490 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.493 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.494 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.521559+0000", 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.521559+0000", 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521848+0000", 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.495 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.496 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2d", 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.497 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:21:09.742073+0000", 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.498 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.501 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.502 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.514291+0000", 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.514291+0000", 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515094+0000", 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.503 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.504 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2c", 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.505 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:39:52.842240+0000", 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.506 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.509 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.510 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.510485+0000", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.510485+0000", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.510758+0000", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:23.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.512 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2b", 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.513 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:26:25.102759+0000", 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.517 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.518 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511289+0000", 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511289+0000", 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.511696+0000", 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.520 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2a", 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.521 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:02:27.657439+0000", 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.525 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.526 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.517314+0000", 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.517314+0000", 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517564+0000", 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.528 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.29", 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.529 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:55:08.360046+0000", 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.530 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.531 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.532 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.533 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.534 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.513008+0000", 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.513008+0000", 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513865+0000", 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.535 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.536 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.28", 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.537 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:18:37.581596+0000", 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.538 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.539 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.540 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.541 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.542 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.517252+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.517252+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517481+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:23.543 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.544 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.27", 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:29:15.209238+0000", 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.545 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.546 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.547 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.548 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.549 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.550 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.520764+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.520764+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521201+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.551 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.552 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.26", 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:39:40.688146+0000", 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.553 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.554 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.555 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.556 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.557 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.503906+0000", 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.503906+0000", 2024-09-14T03:37:23.558 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.504156+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:23.559 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.560 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.25", 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:33:57.350618+0000", 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.561 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.562 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.563 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.564 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.565 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:23.566 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.330493+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.330493+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.330855+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.567 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.568 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.24", 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 31, 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:28:06.278074+0000", 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.569 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.570 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.571 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.572 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.573 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.955095+0000", 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.955095+0000", 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.955446+0000", 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:23.574 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.575 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.23", 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.576 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 18, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:56:01.417881+0000", 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.577 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.578 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.579 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.580 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.581 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512998+0000", 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512998+0000", 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513294+0000", 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.582 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.583 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.22", 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.584 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:04:54.262830+0000", 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.585 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.586 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.587 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.588 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.589 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511955+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511955+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512389+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:23.590 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.591 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.21", 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:30:31.454464+0000", 2024-09-14T03:37:23.592 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.593 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.594 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.596 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.597 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519764+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519764+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520047+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:23.598 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.599 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.20", 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:47:55.233617+0000", 2024-09-14T03:37:23.600 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.601 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.604 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.605 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.515374+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.515374+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515661+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.606 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.607 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1a", 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:03:06.066379+0000", 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.608 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.609 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.610 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.612 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.399904+0000", 2024-09-14T03:37:23.613 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.399904+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.400168+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.614 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.615 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1b", 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:48:35.387300+0000", 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.616 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.617 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.618 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.619 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.620 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.513309+0000", 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.513309+0000", 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.514049+0000", 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:23.621 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.622 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1b", 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.623 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:44:14.127299+0000", 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.624 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.625 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.626 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.627 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.628 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405514+0000", 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405514+0000", 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406006+0000", 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.629 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.630 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1a", 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:23.631 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:23:17.405355+0000", 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.632 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.633 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.634 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.635 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.636 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.398766+0000", 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.398766+0000", 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399102+0000", 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:23.637 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.638 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.19", 2024-09-14T03:37:23.639 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:08:01.084059+0000", 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.640 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.641 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.643 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.644 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519825+0000", 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519825+0000", 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520106+0000", 2024-09-14T03:37:23.645 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:23.646 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.18", 2024-09-14T03:37:23.647 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:28:44.762587+0000", 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.648 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.649 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.651 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.652 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405828+0000", 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405828+0000", 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406587+0000", 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.653 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.654 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.18", 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.655 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:05:57.857900+0000", 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.656 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.657 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.659 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.660 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.506311+0000", 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.506311+0000", 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.507004+0000", 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.661 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.662 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.19", 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.663 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:01:33.038366+0000", 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.664 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.667 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.668 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400274+0000", 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400274+0000", 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.402116+0000", 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.669 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.670 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.17", 2024-09-14T03:37:23.671 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:54:46.213173+0000", 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.672 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.675 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.676 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519909+0000", 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519909+0000", 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520187+0000", 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.677 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.678 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.16", 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.679 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:10:32.588302+0000", 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.680 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.683 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.684 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.401698+0000", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.401698+0000", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.402069+0000", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:23.685 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.686 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.16", 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:16:48.483610+0000", 2024-09-14T03:37:23.687 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.688 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.691 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.692 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.515256+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.515256+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515495+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.693 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.694 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.695 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.695 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.17", 2024-09-14T03:37:23.695 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.695 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.695 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.696 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.696 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.696 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:26:02.392960+0000", 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.700 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.701 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.398281+0000", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.398281+0000", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.398566+0000", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:23.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 3, 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.703 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 3, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.15", 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 27, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.704 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:00:42.503990+0000", 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 90, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 4, 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.708 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'3" 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.709 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.953335+0000", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.953335+0000", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.953658+0000", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:23.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.711 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.14", 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 18, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.712 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:10:40.796694+0000", 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.716 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.717 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400453+0000", 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400453+0000", 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.401007+0000", 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 12, 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.719 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 12, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.14", 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 36, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:07:32.633267+0000", 2024-09-14T03:37:23.720 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 2624, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 12, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 4, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 4, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.723 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 12, 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 9 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "68'12" 2024-09-14T03:37:23.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511328+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511328+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.511714+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.726 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.727 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.15", 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:00:06.445335+0000", 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.732 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.395835+0000", 2024-09-14T03:37:23.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.395835+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.396984+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:23.734 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.735 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.13", 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:49:57.889158+0000", 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.736 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 34, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.739 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 6 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.740 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.741 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.520704+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.520704+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521192+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.742 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.743 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.12", 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:37:13.469199+0000", 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.746 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.747 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.748 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405882+0000", 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405882+0000", 2024-09-14T03:37:23.749 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406737+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:23.750 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.12", 2024-09-14T03:37:23.751 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:00:58.379549+0000", 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.754 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.755 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.756 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516007+0000", 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516007+0000", 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517299+0000", 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.757 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.758 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.13", 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.759 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:16:49.173576+0000", 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.762 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.763 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.764 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400073+0000", 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400073+0000", 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.401544+0000", 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.765 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.766 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.11", 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.767 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:48:24.935055+0000", 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.769 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.770 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.771 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.772 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516657+0000", 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516657+0000", 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516939+0000", 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.773 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.774 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.10", 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.775 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:56:17.423269+0000", 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.777 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.778 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.779 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.780 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400323+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400323+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.400570+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:23.781 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 8, 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.782 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 8, 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.10", 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 32, 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:39:46.484991+0000", 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.783 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 12, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 4, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 4, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.785 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 4, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.786 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 12, 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'8" 2024-09-14T03:37:23.787 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.788 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519973+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519973+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521031+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:23.789 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.790 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.11", 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:12:20.217955+0000", 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.791 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.792 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.793 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.794 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.795 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.398533+0000", 2024-09-14T03:37:23.796 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.398533+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399709+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.797 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.798 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.f", 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:12:43.525175+0000", 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 22, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.801 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 1, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.802 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.803 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.520365+0000", 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.520365+0000", 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520629+0000", 2024-09-14T03:37:23.804 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:23.805 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.806 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.e", 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:59:29.519783+0000", 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.807 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.808 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.809 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.810 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.811 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.404261+0000", 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.404261+0000", 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.404858+0000", 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.812 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:23.813 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.e", 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.814 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:24:38.834576+0000", 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.815 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.816 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.817 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.818 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.819 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516527+0000", 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516527+0000", 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516816+0000", 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.820 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.821 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.f", 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.822 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:49:35.627973+0000", 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.823 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.824 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.825 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.826 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.827 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.395255+0000", 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.395255+0000", 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.395532+0000", 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.828 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.829 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.d", 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.830 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:36:51.477913+0000", 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 70, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.831 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.832 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.833 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 1, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.834 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.835 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.520420+0000", 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.520420+0000", 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520744+0000", 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.836 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.837 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.c", 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.838 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:32:43.573943+0000", 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.839 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.840 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.841 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.842 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.843 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.399008+0000", 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.399008+0000", 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399299+0000", 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.844 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.845 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.c", 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.846 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:16:14.935721+0000", 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.847 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.848 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.849 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.850 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.851 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519164+0000", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519164+0000", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521443+0000", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:23.852 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.853 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.d", 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:34:28.199628+0000", 2024-09-14T03:37:23.854 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.855 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.856 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.859 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.000396+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.000396+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.917557+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.860 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.861 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.5", 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:18:00.378690+0000", 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000302743, 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.862 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.863 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.864 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.865 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.866 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 20, 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:36.967695+0000", 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:36.967695+0000", 2024-09-14T03:37:23.867 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:36.972974+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:31:39.719769+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:31:39.719769+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 51, 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:31:39.719769+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:23.868 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 95, 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 50, 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 95, 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.869 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "1.0", 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 116, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:28:24.258573+0000", 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 590368, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.870 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 590368, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.871 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 2, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.872 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 272, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 479, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 145, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1628 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.873 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "68'95" 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.874 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405473+0000", 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405473+0000", 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406222+0000", 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:23.875 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.876 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.3", 2024-09-14T03:37:23.877 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:09:26.326984+0000", 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.878 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.879 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.881 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.882 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512135+0000", 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512135+0000", 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512416+0000", 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.883 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.884 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2", 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.885 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:31:49.847882+0000", 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.886 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.887 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.889 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.890 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.346558+0000", 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.346558+0000", 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.831437+0000", 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.891 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.892 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.6", 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:23.893 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:15:11.239970+0000", 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000503977, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.894 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.895 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.897 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:23.898 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.901 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.901 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.901 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.901 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.901 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:23.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.408622+0000", 2024-09-14T03:37:23.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.408622+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.409150+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.903 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.0", 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:11:03.508216+0000", 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.907 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.908 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.510158+0000", 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.510158+0000", 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517068+0000", 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:23.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.910 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1", 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.911 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:20:22.709122+0000", 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.915 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.916 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.006563+0000", 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.006563+0000", 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.837898+0000", 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.917 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.918 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.7", 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.919 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:50:37.308715+0000", 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000272495, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.920 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.923 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.924 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.399698+0000", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.399698+0000", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399987+0000", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:23.925 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.926 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1", 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:26:58.806927+0000", 2024-09-14T03:37:23.927 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.931 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.932 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.515012+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.515012+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515266+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:23.933 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.934 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.0", 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:52:23.510282+0000", 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.935 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.936 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.937 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.939 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:23.940 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.346774+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.346774+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.837741+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.941 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.942 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.4", 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:36:53.687838+0000", 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000654522, 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.943 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.944 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.945 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.947 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:23.948 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.399434+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.399434+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399736+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.949 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.950 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.2", 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:28:12.790258+0000", 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.951 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.952 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.953 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.954 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.955 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.956 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.507452+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.507452+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.507742+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.957 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.958 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3", 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:49:50.407482+0000", 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.959 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.960 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.961 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.962 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.963 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.005925+0000", 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.005925+0000", 2024-09-14T03:37:23.964 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.837653+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:23.965 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.966 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.2", 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:57:41.107409+0000", 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000833497, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 19, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.967 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.968 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.969 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 2, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 2 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.970 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "67'2" 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.971 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400204+0000", 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400204+0000", 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.400498+0000", 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:23.972 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.973 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.4", 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.974 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:07:48.735887+0000", 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.975 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.976 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.977 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.978 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.979 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.513689+0000", 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.513689+0000", 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513933+0000", 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:23.980 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.981 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.5", 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.982 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:37:23.791135+0000", 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.983 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:23.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.985 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.986 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:23.987 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.005945+0000", 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.005945+0000", 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.831590+0000", 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.988 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.989 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.3", 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.990 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 20, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:05:00.810273+0000", 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000581876, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.991 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:23.992 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:23.993 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:23.994 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "65'1" 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:23.995 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.954209+0000", 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.954209+0000", 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.954630+0000", 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.996 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:23.997 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.5", 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:23.998 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 18, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:09:57.878910+0000", 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:23.999 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.000 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.001 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.002 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.003 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512679+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512679+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513000+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:24.004 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.005 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.4", 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:43:48.821069+0000", 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.006 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.007 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.008 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.009 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.010 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:51.997457+0000", 2024-09-14T03:37:24.011 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:51.997457+0000", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.860310+0000", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.012 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.013 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.0", 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:38:30.409340+0000", 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000706684, 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.014 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.015 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.016 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.017 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.018 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:24.019 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.397377+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.397377+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.397629+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.020 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:24.021 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.6", 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:42:02.914679+0000", 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.022 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.023 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.024 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 1, 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.025 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.026 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.517367+0000", 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.517367+0000", 2024-09-14T03:37:24.027 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517598+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:24.028 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.029 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.7", 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:32:44.727080+0000", 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.030 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.031 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.032 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.033 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.034 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.006158+0000", 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.006158+0000", 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.830903+0000", 2024-09-14T03:37:24.035 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:24.036 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.037 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.1", 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:33:03.846292+0000", 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000598326, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.038 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.039 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.040 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.041 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:24.042 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.404994+0000", 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.404994+0000", 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.405301+0000", 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.043 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.044 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.7", 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.045 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:52:19.415253+0000", 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.046 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:24.047 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.048 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.049 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.050 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516212+0000", 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516212+0000", 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516570+0000", 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.051 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.052 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.6", 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.053 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:33:23.282986+0000", 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.054 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.055 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.056 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.057 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.058 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.408521+0000", 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.408521+0000", 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.408805+0000", 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.059 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.060 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.8", 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:24.061 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:20:00.847563+0000", 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.062 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.063 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.064 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.065 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.066 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511131+0000", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511131+0000", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.511411+0000", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:24.067 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.068 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.9", 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.069 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:29:23.033860+0000", 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.070 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.071 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.072 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.073 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:24.074 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400172+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400172+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.400483+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.075 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.076 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.9", 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:23:40.756478+0000", 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.077 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.078 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.079 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.080 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.081 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.505460+0000", 2024-09-14T03:37:24.082 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.505460+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.505712+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:24.083 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.084 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.8", 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:08:20.752579+0000", 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.085 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.086 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.087 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.088 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.089 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405382+0000", 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405382+0000", 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.405640+0000", 2024-09-14T03:37:24.090 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:24.091 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.092 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.a", 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:49:36.350128+0000", 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.093 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.096 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.097 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516335+0000", 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516335+0000", 2024-09-14T03:37:24.098 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516581+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:24.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.100 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.b", 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:27:06.070599+0000", 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.101 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.104 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:24.105 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.408264+0000", 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.408264+0000", 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.408573+0000", 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.106 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.107 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.b", 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.108 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:40:30.937449+0000", 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.109 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.110 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.111 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 1, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.112 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:24.113 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516119+0000", 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516119+0000", 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516408+0000", 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.114 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.115 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.a", 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.116 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:38:16.570920+0000", 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.117 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.118 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.120 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.121 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512183+0000", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512183+0000", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512557+0000", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:24.122 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.123 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1d", 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.124 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:05:14.073579+0000", 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.125 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.126 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.128 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.129 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.406065+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.406065+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406333+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.130 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:24.131 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1c", 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:22:04.009889+0000", 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.132 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.133 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.134 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.135 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:24.136 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.508086+0000", 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.508086+0000", 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.508363+0000", 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.137 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.138 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1c", 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.139 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:46:47.194036+0000", 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.140 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.141 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:24.143 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.144 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.406662+0000", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.406662+0000", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406945+0000", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:24.145 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.146 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1d", 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.147 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:46:42.749128+0000", 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.148 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.149 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.151 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.152 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.510663+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.510663+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.510922+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.153 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.154 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1f", 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:00:02.779787+0000", 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.155 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.156 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.157 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.159 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:24.160 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.398725+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.398725+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.398992+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.161 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.162 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1e", 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:50:20.745864+0000", 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.163 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.164 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.165 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.166 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.167 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.514500+0000", 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.514500+0000", 2024-09-14T03:37:24.168 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.514810+0000", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:24.169 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.170 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1e", 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:20:21.110046+0000", 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.171 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.172 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.175 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:24.176 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.396670+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.396670+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.397898+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:24.177 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 4, 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:24.178 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 4, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1f", 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 28, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:15:46.652243+0000", 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:24.179 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 570, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.180 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:24.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 4, 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1 2024-09-14T03:37:24.182 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'4" 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_per_pool": [ 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.183 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 24, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd" 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 3, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pg": 8, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 8, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 3, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "poolid": 4, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 19, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:24.185 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.186 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 2, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 2 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 12288, 2024-09-14T03:37:24.187 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 57, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 24 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.188 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 192, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pg": 64, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 8, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "poolid": 3, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 0, 2024-09-14T03:37:24.192 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 0, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 192 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.193 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 96, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 45, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pg": 32, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 8, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 45, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "poolid": 2, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 3410, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.194 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 66, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 22, 2024-09-14T03:37:24.195 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 22, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 14, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.196 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 51, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 19 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 98304, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:24.197 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 10230, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 96 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 3, 2024-09-14T03:37:24.198 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr" 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 95, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pg": 1, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 3, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 95, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "poolid": 1, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 590368, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 590368, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.200 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 2, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.201 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 272, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 479, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 145, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1628 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 1781760, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:24.202 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 1771104, 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 3 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "usage": { 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 105, 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "pools": 4, 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_by_class": { 2024-09-14T03:37:24.203 INFO:tasks.workunit.client.0.smithi110.stdout: "ssd": { 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: "total_avail_bytes": 765516926976, 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: "total_bytes": 767960285184, 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_bytes": 2443358208, 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_raw_bytes": 2443358208, 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_raw_ratio": 0.0031816205009818077 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: "total_avail_bytes": 765516926976, 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: "total_bytes": 767960285184, 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_bytes": 2443358208 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout:} 2024-09-14T03:37:24.204 INFO:tasks.workunit.client.0.smithi110.stdout:Device report is generated separately. To see it run 'ceph telemetry show-device'. 2024-09-14T03:37:24.205 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph telemetry show-device 2024-09-14T03:37:24.206 INFO:tasks.workunit.client.0.smithi110.stdout:{ 2024-09-14T03:37:24.206 INFO:tasks.workunit.client.0.smithi110.stdout: "a5230552-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:24.206 INFO:tasks.workunit.client.0.smithi110.stdout: "INTEL_SSDPED1D480GA_a523a9c6-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:24.206 INFO:tasks.workunit.client.0.smithi110.stdout: "20240914-033141": { 2024-09-14T03:37:24.206 INFO:tasks.workunit.client.0.smithi110.stdout: "device": { 2024-09-14T03:37:24.206 INFO:tasks.workunit.client.0.smithi110.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: "protocol": "NVMe", 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "nvme" 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: "firmware_version": "E2010325", 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: "host_id": "a5230552-724a-11ef-a07b-0cc47a8fcc35", 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: "json_format_version": [ 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: "local_time": { 2024-09-14T03:37:24.207 INFO:tasks.workunit.client.0.smithi110.stdout: "asctime": "Sat Sep 14 03:31:41 2024 UTC", 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "time_t": 1726284701 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "logical_block_size": 512, 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_controller_id": 0, 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_namespaces": [ 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "capacity": { 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.208 INFO:tasks.workunit.client.0.smithi110.stdout: "formatted_lba_size": 512, 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 1, 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "size": { 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "utilization": { 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_pci_vendor": { 2024-09-14T03:37:24.209 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 32902, 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "subsystem_id": 32902 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "Device stats": { 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_fw_downgrades": { 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "crc_error_count": { 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.210 INFO:tasks.workunit.client.0.smithi110.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "erase_fail_count": { 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "host_bytes_written": { 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 9651523 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "host_ctx_wear_used": { 2024-09-14T03:37:24.211 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "media_bytes_read": { 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "nand_bytes_written": { 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_stat_indicator": { 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.212 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "pll_lock_loss_count": { 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "program_fail_count": { 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.213 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "soft_ecc_err_rate": { 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "thermal_throttle_status": { 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "cnt": 0, 2024-09-14T03:37:24.214 INFO:tasks.workunit.client.0.smithi110.stdout: "pct": 0 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_host_reads": { 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 14 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_media_wear": { 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": "0.000000" 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_timer": { 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 1145676 2024-09-14T03:37:24.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "unexp_power_loss": { 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "wear_leveling": { 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "avg": 0, 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "max": 0, 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.216 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare": 100, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare_threshold": 0, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "controller_busy_time": 3744, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "critical_warning": 0, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_read": 130374666, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_written": 632522258, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "host_reads": 2081328881, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "host_writes": 14581181920, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "media_errors": 0, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "num_err_log_entries": 0, 2024-09-14T03:37:24.217 INFO:tasks.workunit.client.0.smithi110.stdout: "percentage_used": 0, 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycles": 25695, 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_hours": 30070, 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": 32, 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "unsafe_shutdowns": 8731 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_vendor": "intel", 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_version": { 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "string": "<1.2", 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycle_count": 25695, 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_time": { 2024-09-14T03:37:24.218 INFO:tasks.workunit.client.0.smithi110.stdout: "hours": 30070 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "serial_number": "deleted", 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "smart_status": { 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme": { 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "passed": true 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl": { 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "argv": [ 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl", 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "-x", 2024-09-14T03:37:24.219 INFO:tasks.workunit.client.0.smithi110.stdout: "--json=o", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "/dev/nvme0n1" 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "build_info": "(local build)", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "exit_status": 0, 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "output": [ 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "Serial Number: deleted", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Version: E2010325", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:37:24.220 INFO:tasks.workunit.client.0.smithi110.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller ID: 0", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "NVMe Version: <1.2", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Number of Namespaces: 1", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Local Time is: Sat Sep 14 03:31:41 2024 UTC", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.221 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported Power States", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 512 0 2", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "Critical Warning: 0x00", 2024-09-14T03:37:24.222 INFO:tasks.workunit.client.0.smithi110.stdout: "Temperature: 32 Celsius", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare: 100%", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Percentage Used: 0%", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Read: 130,374,666 [66.7 TB]", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Written: 632,522,258 [323 TB]", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Read Commands: 2,081,328,881", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Write Commands: 14,581,181,920", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller Busy Time: 3,744", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Power Cycles: 25,695", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Power On Hours: 30,070", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Unsafe Shutdowns: 8,731", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information Log Entries: 0", 2024-09-14T03:37:24.223 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: "No Errors Logged", 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: "" 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: "svn_revision": "5155", 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: "version": [ 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": { 2024-09-14T03:37:24.224 INFO:tasks.workunit.client.0.smithi110.stdout: "current": 32 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "user_capacity": { 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "a524ce46-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "INTEL_SSDPED1D480GA_a52546be-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "20240914-033143": { 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "device": { 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:37:24.225 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: "protocol": "NVMe", 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "nvme" 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: "firmware_version": "E2010325", 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: "host_id": "a524ce46-724a-11ef-a07b-0cc47a8fcc35", 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: "json_format_version": [ 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: "local_time": { 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: "asctime": "Sat Sep 14 03:31:43 2024 UTC", 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: "time_t": 1726284703 2024-09-14T03:37:24.226 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "logical_block_size": 512, 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_controller_id": 0, 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_namespaces": [ 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "capacity": { 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "formatted_lba_size": 512, 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 1, 2024-09-14T03:37:24.227 INFO:tasks.workunit.client.0.smithi110.stdout: "size": { 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "utilization": { 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_pci_vendor": { 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 32902, 2024-09-14T03:37:24.228 INFO:tasks.workunit.client.0.smithi110.stdout: "subsystem_id": 32902 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "Device stats": { 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_fw_downgrades": { 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "crc_error_count": { 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:37:24.229 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "erase_fail_count": { 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "host_bytes_written": { 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 10171674 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "host_ctx_wear_used": { 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.230 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "media_bytes_read": { 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "nand_bytes_written": { 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_stat_indicator": { 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.231 INFO:tasks.workunit.client.0.smithi110.stdout: "pll_lock_loss_count": { 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "program_fail_count": { 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.232 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "soft_ecc_err_rate": { 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "thermal_throttle_status": { 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "cnt": 0, 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "pct": 0 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.233 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_host_reads": { 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 15 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_media_wear": { 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": "0.000000" 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_timer": { 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 1390641 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "unexp_power_loss": { 2024-09-14T03:37:24.234 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "wear_leveling": { 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "avg": 0, 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "max": 0, 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.235 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare": 100, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare_threshold": 0, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "controller_busy_time": 3961, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "critical_warning": 0, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_read": 141031320, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_written": 666610881, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "host_reads": 2714455028, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "host_writes": 13997711582, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "media_errors": 0, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_err_log_entries": 0, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "percentage_used": 0, 2024-09-14T03:37:24.236 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycles": 35342, 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_hours": 38021, 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": 33, 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "unsafe_shutdowns": 11022 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_vendor": "intel", 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_version": { 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "string": "<1.2", 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycle_count": 35342, 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_time": { 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "hours": 38021 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.237 INFO:tasks.workunit.client.0.smithi110.stdout: "serial_number": "deleted", 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "smart_status": { 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme": { 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "passed": true 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl": { 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "argv": [ 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl", 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "-x", 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "--json=o", 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "/dev/nvme0n1" 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.238 INFO:tasks.workunit.client.0.smithi110.stdout: "build_info": "(local build)", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "exit_status": 0, 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "output": [ 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "Serial Number: deleted", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Version: E2010325", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller ID: 0", 2024-09-14T03:37:24.239 INFO:tasks.workunit.client.0.smithi110.stdout: "NVMe Version: <1.2", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Number of Namespaces: 1", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Local Time is: Sat Sep 14 03:31:43 2024 UTC", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported Power States", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:37:24.240 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 512 0 2", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "Critical Warning: 0x00", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "Temperature: 33 Celsius", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare: 100%", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:37:24.241 INFO:tasks.workunit.client.0.smithi110.stdout: "Percentage Used: 0%", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Read: 141,031,320 [72.2 TB]", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Written: 666,610,881 [341 TB]", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Read Commands: 2,714,455,028", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Write Commands: 13,997,711,582", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller Busy Time: 3,961", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Power Cycles: 35,342", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Power On Hours: 38,021", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Unsafe Shutdowns: 11,022", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information Log Entries: 0", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:37:24.242 INFO:tasks.workunit.client.0.smithi110.stdout: "No Errors Logged", 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: "" 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: "svn_revision": "5155", 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: "version": [ 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": { 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: "current": 33 2024-09-14T03:37:24.243 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.244 INFO:tasks.workunit.client.0.smithi110.stdout: "user_capacity": { 2024-09-14T03:37:24.244 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.244 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.244 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.244 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.244 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.244 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.244 INFO:tasks.workunit.client.0.smithi110.stdout:} 2024-09-14T03:37:24.244 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.246 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.247 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.248 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.249 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.249 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:22 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.249 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph telemetry show-all 2024-09-14T03:37:24.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:24.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.249 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.250 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.253 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:22 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:24.264 INFO:tasks.workunit.client.0.smithi110.stdout:{ 2024-09-14T03:37:24.264 INFO:tasks.workunit.client.0.smithi110.stdout: "device_report": { 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "a5230552-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "INTEL_SSDPED1D480GA_a523a9c6-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "20240914-033141": { 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "device": { 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "protocol": "NVMe", 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "nvme" 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "firmware_version": "E2010325", 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "host_id": "a5230552-724a-11ef-a07b-0cc47a8fcc35", 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: "json_format_version": [ 2024-09-14T03:37:24.265 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "local_time": { 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "asctime": "Sat Sep 14 03:31:41 2024 UTC", 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "time_t": 1726284701 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "logical_block_size": 512, 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_controller_id": 0, 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_namespaces": [ 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "capacity": { 2024-09-14T03:37:24.266 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "formatted_lba_size": 512, 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 1, 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "size": { 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "utilization": { 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.267 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_pci_vendor": { 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 32902, 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "subsystem_id": 32902 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "Device stats": { 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_fw_downgrades": { 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.268 INFO:tasks.workunit.client.0.smithi110.stdout: "crc_error_count": { 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "erase_fail_count": { 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.269 INFO:tasks.workunit.client.0.smithi110.stdout: "host_bytes_written": { 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 9651523 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "host_ctx_wear_used": { 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "media_bytes_read": { 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "nand_bytes_written": { 2024-09-14T03:37:24.270 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_stat_indicator": { 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "pll_lock_loss_count": { 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "program_fail_count": { 2024-09-14T03:37:24.271 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "soft_ecc_err_rate": { 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.272 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: "thermal_throttle_status": { 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: "cnt": 0, 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pct": 0 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_host_reads": { 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 14 2024-09-14T03:37:24.273 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_media_wear": { 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": "0.000000" 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_timer": { 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 1145676 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "unexp_power_loss": { 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.274 INFO:tasks.workunit.client.0.smithi110.stdout: "wear_leveling": { 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "avg": 0, 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "max": 0, 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare": 100, 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare_threshold": 0, 2024-09-14T03:37:24.275 INFO:tasks.workunit.client.0.smithi110.stdout: "controller_busy_time": 3744, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "critical_warning": 0, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_read": 130374666, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_written": 632522258, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "host_reads": 2081328881, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "host_writes": 14581181920, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "media_errors": 0, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_err_log_entries": 0, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "percentage_used": 0, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycles": 25695, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_hours": 30070, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": 32, 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: "unsafe_shutdowns": 8731 2024-09-14T03:37:24.276 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_vendor": "intel", 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_version": { 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "string": "<1.2", 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycle_count": 25695, 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_time": { 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "hours": 30070 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "serial_number": "deleted", 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "smart_status": { 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme": { 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "passed": true 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl": { 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "argv": [ 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl", 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "-x", 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "--json=o", 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "/dev/nvme0n1" 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "build_info": "(local build)", 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "exit_status": 0, 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "output": [ 2024-09-14T03:37:24.278 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "Serial Number: deleted", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Version: E2010325", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller ID: 0", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "NVMe Version: <1.2", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "Number of Namespaces: 1", 2024-09-14T03:37:24.279 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Local Time is: Sat Sep 14 03:31:41 2024 UTC", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported Power States", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.280 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 512 0 2", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "Critical Warning: 0x00", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "Temperature: 32 Celsius", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare: 100%", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "Percentage Used: 0%", 2024-09-14T03:37:24.281 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Read: 130,374,666 [66.7 TB]", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Written: 632,522,258 [323 TB]", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Read Commands: 2,081,328,881", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Write Commands: 14,581,181,920", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller Busy Time: 3,744", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Power Cycles: 25,695", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Power On Hours: 30,070", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Unsafe Shutdowns: 8,731", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information Log Entries: 0", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "No Errors Logged", 2024-09-14T03:37:24.282 INFO:tasks.workunit.client.0.smithi110.stdout: "" 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: "svn_revision": "5155", 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: "version": [ 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": { 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: "current": 32 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: "user_capacity": { 2024-09-14T03:37:24.283 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "a524ce46-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "INTEL_SSDPED1D480GA_a52546be-724a-11ef-a07b-0cc47a8fcc35": { 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "20240914-033143": { 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "device": { 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "protocol": "NVMe", 2024-09-14T03:37:24.284 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "nvme" 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: "firmware_version": "E2010325", 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: "host_id": "a524ce46-724a-11ef-a07b-0cc47a8fcc35", 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: "json_format_version": [ 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: "local_time": { 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: "asctime": "Sat Sep 14 03:31:43 2024 UTC", 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: "time_t": 1726284703 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: "logical_block_size": 512, 2024-09-14T03:37:24.285 INFO:tasks.workunit.client.0.smithi110.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_controller_id": 0, 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_namespaces": [ 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "capacity": { 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "formatted_lba_size": 512, 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 1, 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "size": { 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.286 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: "utilization": { 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_pci_vendor": { 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: "id": 32902, 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: "subsystem_id": 32902 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.287 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "Device stats": { 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_fw_downgrades": { 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "crc_error_count": { 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.288 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "erase_fail_count": { 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "host_bytes_written": { 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 10171674 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "host_ctx_wear_used": { 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.289 INFO:tasks.workunit.client.0.smithi110.stdout: "media_bytes_read": { 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "nand_bytes_written": { 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_stat_indicator": { 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "pll_lock_loss_count": { 2024-09-14T03:37:24.290 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "program_fail_count": { 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.291 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "soft_ecc_err_rate": { 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "thermal_throttle_status": { 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "cnt": 0, 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "pct": 0 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.292 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_host_reads": { 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 15 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_media_wear": { 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": "0.000000" 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "timed_workload_timer": { 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 100, 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 1390641 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "unexp_power_loss": { 2024-09-14T03:37:24.293 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": 0 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: "wear_leveling": { 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: "normalized": 0, 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: "raw": { 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: "avg": 0, 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: "max": 0, 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:37:24.294 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare": 100, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "available_spare_threshold": 0, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "controller_busy_time": 3961, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "critical_warning": 0, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_read": 141031320, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "data_units_written": 666610881, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "host_reads": 2714455028, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "host_writes": 13997711582, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "media_errors": 0, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "num_err_log_entries": 0, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "percentage_used": 0, 2024-09-14T03:37:24.295 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycles": 35342, 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_hours": 38021, 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": 33, 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "unsafe_shutdowns": 11022 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_vendor": "intel", 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme_version": { 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "string": "<1.2", 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "power_cycle_count": 35342, 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "power_on_time": { 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: "hours": 38021 2024-09-14T03:37:24.296 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "serial_number": "deleted", 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "smart_status": { 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "nvme": { 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "passed": true 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl": { 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "argv": [ 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl", 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "-x", 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "--json=o", 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: "/dev/nvme0n1" 2024-09-14T03:37:24.297 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "build_info": "(local build)", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "exit_status": 0, 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "output": [ 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "Serial Number: deleted", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Version: E2010325", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:37:24.298 INFO:tasks.workunit.client.0.smithi110.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller ID: 0", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "NVMe Version: <1.2", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Number of Namespaces: 1", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Local Time is: Sat Sep 14 03:31:43 2024 UTC", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.299 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported Power States", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: " 0 + 512 0 2", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "Critical Warning: 0x00", 2024-09-14T03:37:24.300 INFO:tasks.workunit.client.0.smithi110.stdout: "Temperature: 33 Celsius", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare: 100%", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Percentage Used: 0%", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Read: 141,031,320 [72.2 TB]", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Data Units Written: 666,610,881 [341 TB]", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Read Commands: 2,714,455,028", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Host Write Commands: 13,997,711,582", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Controller Busy Time: 3,961", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Power Cycles: 35,342", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Power On Hours: 38,021", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Unsafe Shutdowns: 11,022", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:37:24.301 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information Log Entries: 0", 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "", 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "No Errors Logged", 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "" 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "svn_revision": "5155", 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "version": [ 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "temperature": { 2024-09-14T03:37:24.302 INFO:tasks.workunit.client.0.smithi110.stdout: "current": 33 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: "user_capacity": { 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: "blocks": 937703088, 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 480103981056 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: "report": { 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: "balancer": { 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: "active": true, 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: "mode": "" 2024-09-14T03:37:24.303 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "channels": [ 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "basic", 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "crash", 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "device", 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "ident", 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "perf" 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "channels_available": [ 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "basic", 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "ident", 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "crash", 2024-09-14T03:37:24.304 INFO:tasks.workunit.client.0.smithi110.stdout: "device", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "perf" 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "collections_available": [ 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_base", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "device_base", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "crash_base", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "ident_base", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_perf", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_mds_metadata", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_usage", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_usage_by_class", 2024-09-14T03:37:24.305 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_rook_v01", 2024-09-14T03:37:24.308 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_memory_metrics", 2024-09-14T03:37:24.308 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_options_bluestore" 2024-09-14T03:37:24.308 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.308 INFO:tasks.workunit.client.0.smithi110.stdout: "collections_opted_in": [ 2024-09-14T03:37:24.308 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_base", 2024-09-14T03:37:24.308 INFO:tasks.workunit.client.0.smithi110.stdout: "device_base", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "crash_base", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "ident_base", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_perf", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_mds_metadata", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_usage", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_usage_by_class", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_rook_v01", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_memory_metrics", 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "basic_pool_options_bluestore" 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "config": { 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "active_changed": [ 2024-09-14T03:37:24.309 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_debug", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_service_ticket_ttl", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "bdev_debug_aio", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "client_check_pool_perm", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "client_mount_gid", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "client_mount_uid", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "client_quota", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "container_image", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "daemonize", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mgr", 2024-09-14T03:37:24.310 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mon", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_ms", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_osd", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_paxos", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "keyring", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_block_size", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_cache_size", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_compression", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_log", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "leveldb_write_buffer_size", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_file", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_journald", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_stderr", 2024-09-14T03:37:24.311 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_allow_pool_delete", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_clock_drift_allowed", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_file", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_journald", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_stderr", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_data_avail_warn", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_election_default_strategy", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_host", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_max_pg_per_osd", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:37:24.312 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_bug", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_old_message", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "no_config_file", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_default_list", 2024-09-14T03:37:24.313 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_load_list", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_op_order", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_shutdown", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_profile", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-14T03:37:24.314 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_lim", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_res", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target_autotune", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue_cut_off", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recover_clone_overlap", 2024-09-14T03:37:24.315 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recovery_max_chunk", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_load_threshold", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_max_interval", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "public_addr", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "public_addrv", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "public_network", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd_default_features", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd_qos_exclude_ops", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "setgroup", 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "setuser" 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster_changed": [ 2024-09-14T03:37:24.316 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_debug", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "auth_service_ticket_ttl", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "bdev_debug_aio", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "container_image", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mgr", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_mon", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_ms", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_osd", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "debug_paxos", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_file", 2024-09-14T03:37:24.317 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_journald", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "log_to_stderr", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_join_fs", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/allow_ptrace", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/container_init", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/migration_current", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/cephadm/use_repo_digest", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/dashboard/ssl_server_port", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/orchestrator/orchestrator", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/channel_ident", 2024-09-14T03:37:24.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/channel_perf", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/description", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/enabled", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr/telemetry/organization", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_allow_pool_delete", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_clock_drift_allowed", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_cluster_log_to_file", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_data_avail_warn", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_election_default_strategy", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_max_pg_per_osd", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:37:24.319 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_bug", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "ms_die_on_old_message", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_default_list", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_class_load_list", 2024-09-14T03:37:24.320 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_op_order", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_shutdown", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mclock_profile", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_memory_target_autotune", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_op_queue_cut_off", 2024-09-14T03:37:24.321 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recover_clone_overlap", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_recovery_max_chunk", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_load_threshold", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_scrub_max_interval", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "public_network", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_cache_enabled", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_enable_ops_log", 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "rgw_enable_usage_log" 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "contact": null, 2024-09-14T03:37:24.322 INFO:tasks.workunit.client.0.smithi110.stdout: "crashes": [], 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "created": "2024-09-14T03:28:36.548032Z", 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "crush": { 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_algs": { 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "straw2": 3 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_sizes": { 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "2": 1, 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "4": 2 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket_types": { 2024-09-14T03:37:24.323 INFO:tasks.workunit.client.0.smithi110.stdout: "1": 2, 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "11": 1 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "compat_weight_set": false, 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "device_classes": [ 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: 8 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "num_buckets": 6, 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "num_devices": 8, 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "num_rules": 1, 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "num_types": 12, 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "num_weight_sets": 0, 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "tunables": { 2024-09-14T03:37:24.324 INFO:tasks.workunit.client.0.smithi110.stdout: "allowed_bucket_algs": 54, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_local_fallback_tries": 0, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_local_tries": 0, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "choose_total_tries": 50, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_descend_once": 1, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_stable": 1, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "chooseleaf_vary_r": 1, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v2_rules": 0, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v3_rules": 0, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v4_buckets": 1, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "has_v5_rules": 0, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "legacy_tunables": 0, 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "minimum_required_version": "jewel", 2024-09-14T03:37:24.325 INFO:tasks.workunit.client.0.smithi110.stdout: "optimal_tunables": 1, 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "profile": "jewel", 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables": 1, 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables2": 1, 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables3": 1, 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "require_feature_tunables5": 1, 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "straw_calc_version": 1 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "description": "upgrade test cluster", 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "fs": { 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "feature_flags": { 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "enable_multiple": true, 2024-09-14T03:37:24.326 INFO:tasks.workunit.client.0.smithi110.stdout: "ever_enabled_multiple": true 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "filesystems": [ 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "approx_ctime": "2024-09", 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "balancer_enabled": false, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_caps": 0, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_dns": 10, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_inos": 13, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "cached_subtrees": 2, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "ever_allowed_features": 0, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "explicitly_allowed_features": 0, 2024-09-14T03:37:24.327 INFO:tasks.workunit.client.0.smithi110.stdout: "files": 0, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "max_mds": 1, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "num_data_pools": 1, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "num_in": 1, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "num_mds": 1, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": 0, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "num_standby_replay": 0, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "num_up": 1, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "snaps": 0, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "standby_count_wanted": 0 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "num_standby_mds": 1, 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: "total_num_mds": 2 2024-09-14T03:37:24.328 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_stats": { 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 20676608, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 440800, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 507904, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 441, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 1249848, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 20, 2024-09-14T03:37:24.329 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1251840, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 15112680, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 21184512 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 20135936, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 336504, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 802816, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 0, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 347, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 956376, 2024-09-14T03:37:24.330 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 12, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 722944, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 14695856, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 20135936 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 46825472, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 1273552, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 786432, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 2752512, 2024-09-14T03:37:24.331 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1780, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 10724672, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 22, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2690048, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 28729328, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 49577984 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 37781504, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 1280272, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 139264, 2024-09-14T03:37:24.332 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 9732096, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1216, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 7648640, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 22, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1769216, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 24322672, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 47513600 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.b030e701d31b8bd7b2036a36a31490ca893f182a": { 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 36552704, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 701168, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2621440, 2024-09-14T03:37:24.333 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 712704, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 9904128, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1020, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 6993000, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 21, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2139904, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 23384488, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 46456832 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.0": { 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 274759680, 2024-09-14T03:37:24.334 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 987720, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 1007616, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 16384, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1370, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 5186528, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2377728, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 262447576, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 274776064 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.1": { 2024-09-14T03:37:24.335 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272310272, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 690792, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 303104, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1312, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 5851248, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1592576, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 261423144, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 272613376 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.336 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.2": { 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272310272, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 705872, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 581632, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1260, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 5116824, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2073600, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 261661464, 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 272891904 2024-09-14T03:37:24.337 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.3": { 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 271532032, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 708744, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 98304, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1231, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4567072, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1814784, 2024-09-14T03:37:24.338 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 261688920, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 271630336 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.4": { 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272801792, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 800088, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 90112, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1232, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4507568, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:37:24.339 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2500864, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 262240760, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 272891904 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.5": { 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 275079168, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 759000, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 4849664, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 0, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 745472, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1269, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4686432, 2024-09-14T03:37:24.340 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2491136, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 262292936, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 275824640 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.6": { 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272957440, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 1055264, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 32768, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 983040, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1251, 2024-09-14T03:37:24.341 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4471064, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 2498304, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 262147528, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 273940480 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.7": { 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "actual_memory_used": 272211968, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "central_cache_freelist": 822304, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "malloc_metadata": 2752512, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "page_heap_freelist": 32768, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "released_to_os": 1515520, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "spans_in_use": 1269, 2024-09-14T03:37:24.342 INFO:tasks.workunit.client.0.smithi110.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_cache_freelists": 4722512, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "transfer_cache_freelist": 1916160, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "use_by_application": 261965712, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "virtual_address_space_used": 273727488 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "hosts": { 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "num": 2, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mds": 2, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mgr": 2, 2024-09-14T03:37:24.343 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_mon": 2, 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "num_with_osd": 2 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "io_rate": { 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_stats_delta": { 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 0, 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 0, 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "stamp_delta": "12.002394", 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:24.344 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:24.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:24.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 146, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 248, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 32, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 0, 2024-09-14T03:37:24.347 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 0, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 0 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.348 INFO:tasks.workunit.client.0.smithi110.stdout: "leaderboard": false, 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "license": "sharing-1-0", 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "mempool": { 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.349 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.350 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.351 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.352 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.353 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 222183, 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 68 2024-09-14T03:37:24.354 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 50488, 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 208 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.355 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 7944, 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 135 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.356 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.357 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.358 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.359 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.360 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.361 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.362 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 149074, 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 33 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.363 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 856, 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 2 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.364 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.365 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.366 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.367 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.368 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.369 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.370 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.371 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1143962, 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 359 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.372 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 352, 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 4 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.373 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6592, 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 134 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12744, 2024-09-14T03:37:24.374 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1546 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 10624, 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 206 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.375 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.376 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.377 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.378 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.379 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.380 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1312422, 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 401 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.381 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.382 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6592, 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 134 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12744, 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1546 2024-09-14T03:37:24.383 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5312, 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 103 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.384 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.b030e701d31b8bd7b2036a36a31490ca893f182a": { 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.385 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.386 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.387 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.388 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.389 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1095110, 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 351 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.390 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.391 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6592, 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 134 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12744, 2024-09-14T03:37:24.392 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1546 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5312, 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 103 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.393 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.0": { 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.394 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20608, 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 112, 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:24.395 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8832, 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20608, 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5840, 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 73 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.396 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4096, 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 54642, 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9929 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 136000, 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 200 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.397 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 11124, 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 713 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 31676, 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 200 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.398 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 426628, 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 93 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2447296, 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 195 2024-09-14T03:37:24.399 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5544, 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 63 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 521280, 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 45 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.400 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 76544, 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 152 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234064, 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3472 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.401 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.402 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.1": { 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:24.403 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19040, 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 896, 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.404 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8160, 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19040, 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4640, 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 58 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.405 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12288, 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 48031, 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8777 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 124440, 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 183 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 9040, 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 534 2024-09-14T03:37:24.406 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 27056, 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 183 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 736, 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.407 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 420108, 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 92 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12096, 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2410957, 2024-09-14T03:37:24.408 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 174 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4840, 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 55 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.409 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 428608, 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 37 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8528, 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 22 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234064, 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3472 2024-09-14T03:37:24.410 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.411 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.2": { 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:24.412 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19040, 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.413 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1120, 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 10 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8160, 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19040, 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 170 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.414 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4400, 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 55 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 26471, 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 6 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 46086, 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8540 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 121040, 2024-09-14T03:37:24.415 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 178 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8368, 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 528 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 25776, 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 178 2024-09-14T03:37:24.416 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 736, 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 358142, 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 77 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12096, 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.417 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2427224, 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 180 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4576, 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 52 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 359104, 2024-09-14T03:37:24.418 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 31 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8320, 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 24 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234576, 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3496 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.419 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.420 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.3": { 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.421 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19600, 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 336, 2024-09-14T03:37:24.422 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8400, 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19600, 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 3440, 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 43 2024-09-14T03:37:24.423 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4096, 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 47348, 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8818 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 125120, 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.424 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 9060, 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 545 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 26472, 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 184 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2208, 2024-09-14T03:37:24.425 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 247335, 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 50 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 30637, 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2405411, 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 179 2024-09-14T03:37:24.426 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4752, 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 54 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 417024, 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 36 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.427 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 10400, 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 20 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234064, 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3472 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.428 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.429 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.4": { 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.430 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19600, 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 560, 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 5 2024-09-14T03:37:24.431 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8400, 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19600, 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 175 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 3440, 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 43 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.432 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 48760, 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9014 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 127840, 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 188 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.433 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 9364, 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 546 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 27496, 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 188 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 736, 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 1 2024-09-14T03:37:24.434 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 895536, 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 65 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 12096, 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2397348, 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 168 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.435 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4752, 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 54 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 451776, 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 39 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.436 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8944, 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 18 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234320, 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3484 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.437 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.438 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.5": { 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.439 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19376, 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 173 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 224, 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 2 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.440 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8304, 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 173 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 19376, 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 173 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2960, 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 37 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.441 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 49818, 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9103 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 129200, 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 190 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 9692, 2024-09-14T03:37:24.442 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 547 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 28344, 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 190 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2208, 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.443 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 812795, 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 45 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 30637, 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2427458, 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 186 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.444 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 5544, 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 63 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 486528, 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 42 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.445 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 13520, 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 26 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234832, 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3508 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.446 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.447 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.6": { 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:24.448 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20384, 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.449 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8736, 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20384, 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2640, 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 33 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.450 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 49123, 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9296 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 127160, 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 187 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 10604, 2024-09-14T03:37:24.451 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 703 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 28604, 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 187 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.452 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 829053, 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 49 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2404108, 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 166 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.453 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 4664, 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 53 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 451776, 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 39 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.454 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 75712, 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 156 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234832, 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3508 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.455 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.456 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.7": { 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "bloom_filter": { 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6304, 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 115 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_reader": { 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 16896, 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 8 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.457 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs_file_writer": { 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 576, 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Blob": { 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20384, 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Buffer": { 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 1232, 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 11 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_Extent": { 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 8736, 2024-09-14T03:37:24.458 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_SharedBlob": { 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 20384, 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 182 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_alloc": { 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2240, 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 28 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_data": { 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 286720, 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 11 2024-09-14T03:37:24.459 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_meta": { 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 53184, 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 9789 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_onode": { 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 133960, 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 197 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_cache_other": { 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 11388, 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 710 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.460 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_fsck": { 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_inline_bl": { 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 31164, 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 197 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_txc": { 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing": { 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 183505, 2024-09-14T03:37:24.461 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 35 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore_writing_deferred": { 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_anon": { 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 2407188, 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 198 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "buffer_meta": { 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 6336, 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 72 2024-09-14T03:37:24.462 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_co": { 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 532864, 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 46 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_mapbl": { 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.463 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_pglog": { 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 80600, 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 199 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap": { 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 234064, 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 3472 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap_mapping": { 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "pgmap": { 2024-09-14T03:37:24.464 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_1": { 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: "unittest_2": { 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes": 0, 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: "items": 0 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.465 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "metadata": { 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 2 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 2 2024-09-14T03:37:24.466 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 2 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 2 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.467 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 3 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 3 2024-09-14T03:37:24.468 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 3 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 3 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.469 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "arch": { 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "x86_64": 8 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_version": { 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "cpu": { 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "distro": { 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "centos": 8 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.470 INFO:tasks.workunit.client.0.smithi110.stdout: "distro_description": { 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "CentOS Stream 9": 8 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_description": { 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "kernel_version": { 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "os": { 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "Linux": 8 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_objectstore": { 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": 8 2024-09-14T03:37:24.471 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "rotational": { 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "0": 8 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "features": { 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "optional": [], 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "persistent": [ 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "kraken", 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "luminous", 2024-09-14T03:37:24.472 INFO:tasks.workunit.client.0.smithi110.stdout: "mimic", 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "osdmap-prune", 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "nautilus", 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "octopus", 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "pacific", 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "elector-pinging", 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "quincy" 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "ipv4_addr_mons": 6, 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "ipv6_addr_mons": 0, 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "min_mon_release": 17, 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "v1_addr_mons": 3, 2024-09-14T03:37:24.473 INFO:tasks.workunit.client.0.smithi110.stdout: "v2_addr_mons": 3 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "organization": "ceph-qa", 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster_network": false, 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "require_min_compat_client": "luminous", 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "require_osd_release": "quincy" 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_perf_histograms": [ 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "axes": [ 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.474 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Latency (usec)", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 100000, 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 99999]", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[100000, 199999]", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[200000, 399999]", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[400000, 799999]", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[800000, 1599999]", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[1600000, 3199999]", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[3200000, 6399999]", 2024-09-14T03:37:24.475 INFO:tasks.workunit.client.0.smithi110.stdout: "[6400000, 12799999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[12800000, 25599999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[25600000, 51199999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[51200000, 102399999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[102400000, 204799999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[204800000, 409599999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[409600000, 819199999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[819200000, 1638399999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[1638400000, 3276799999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[3276800000, 6553599999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[6553600000, 13107199999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[13107200000, 26214399999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[26214400000, 52428799999]", 2024-09-14T03:37:24.476 INFO:tasks.workunit.client.0.smithi110.stdout: "[52428800000, 104857599999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[104857600000, 209715199999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[209715200000, 419430399999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[419430400000, 838860799999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "[53687091200000, None]" 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:24.477 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Request size (bytes)", 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 512, 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 511]", 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "[512, 1023]", 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "[1024, 2047]", 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "[2048, 4095]", 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "[4096, 8191]", 2024-09-14T03:37:24.478 INFO:tasks.workunit.client.0.smithi110.stdout: "[8192, 16383]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[16384, 32767]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[32768, 65535]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[65536, 131071]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[131072, 262143]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[262144, 524287]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[524288, 1048575]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[1048576, 2097151]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[2097152, 4194303]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[4194304, 8388607]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[8388608, 16777215]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[16777216, 33554431]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[33554432, 67108863]", 2024-09-14T03:37:24.479 INFO:tasks.workunit.client.0.smithi110.stdout: "[67108864, 134217727]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[134217728, 268435455]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[268435456, 536870911]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[536870912, 1073741823]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[1073741824, 2147483647]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[2147483648, 4294967295]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[4294967296, 8589934591]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[8589934592, 17179869183]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[17179869184, 34359738367]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[34359738368, 68719476735]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[68719476736, 137438953471]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[137438953472, 274877906943]", 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: "[274877906944, None]" 2024-09-14T03:37:24.480 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "osds": [ 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 0, 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.481 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.482 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.483 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 1, 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.484 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.485 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.486 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 2, 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.487 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.488 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.489 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 3, 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.490 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.491 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 4, 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.492 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.493 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.494 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 5, 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.495 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.496 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.497 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 6, 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.498 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.499 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.500 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 7, 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 118, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.501 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.502 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.503 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: "axes": [ 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Latency (usec)", 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 100000, 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:24.504 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 99999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[100000, 199999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[200000, 399999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[400000, 799999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[800000, 1599999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[1600000, 3199999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[3200000, 6399999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[6400000, 12799999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[12800000, 25599999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[25600000, 51199999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[51200000, 102399999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[102400000, 204799999]", 2024-09-14T03:37:24.505 INFO:tasks.workunit.client.0.smithi110.stdout: "[204800000, 409599999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[409600000, 819199999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[819200000, 1638399999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[1638400000, 3276799999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[3276800000, 6553599999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[6553600000, 13107199999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[13107200000, 26214399999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[26214400000, 52428799999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[52428800000, 104857599999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[104857600000, 209715199999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[209715200000, 419430399999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[419430400000, 838860799999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:37:24.506 INFO:tasks.workunit.client.0.smithi110.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "[53687091200000, None]" 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Request size (bytes)", 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 512, 2024-09-14T03:37:24.507 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 511]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[512, 1023]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[1024, 2047]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[2048, 4095]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[4096, 8191]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[8192, 16383]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[16384, 32767]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[32768, 65535]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[65536, 131071]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[131072, 262143]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[262144, 524287]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[524288, 1048575]", 2024-09-14T03:37:24.508 INFO:tasks.workunit.client.0.smithi110.stdout: "[1048576, 2097151]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[2097152, 4194303]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[4194304, 8388607]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[8388608, 16777215]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[16777216, 33554431]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[33554432, 67108863]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[67108864, 134217727]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[134217728, 268435455]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[268435456, 536870911]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[536870912, 1073741823]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[1073741824, 2147483647]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[2147483648, 4294967295]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[4294967296, 8589934591]", 2024-09-14T03:37:24.509 INFO:tasks.workunit.client.0.smithi110.stdout: "[8589934592, 17179869183]", 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "[17179869184, 34359738367]", 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "[34359738368, 68719476735]", 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "[68719476736, 137438953471]", 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "[137438953472, 274877906943]", 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "[274877906944, None]" 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "osds": [ 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 0, 2024-09-14T03:37:24.510 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.511 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.512 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 1, 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.513 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.514 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.515 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 2, 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.516 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.517 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.518 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 3, 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.519 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.520 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.521 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 4, 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.522 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.523 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.524 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 5, 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.525 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.526 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.527 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 6, 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.528 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.529 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.530 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 7, 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.531 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.532 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.533 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "axes": [ 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Latency (usec)", 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 100000, 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 99999]", 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "[100000, 199999]", 2024-09-14T03:37:24.534 INFO:tasks.workunit.client.0.smithi110.stdout: "[200000, 399999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[400000, 799999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[800000, 1599999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[1600000, 3199999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[3200000, 6399999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[6400000, 12799999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[12800000, 25599999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[25600000, 51199999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[51200000, 102399999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[102400000, 204799999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[204800000, 409599999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[409600000, 819199999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[819200000, 1638399999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[1638400000, 3276799999]", 2024-09-14T03:37:24.535 INFO:tasks.workunit.client.0.smithi110.stdout: "[3276800000, 6553599999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[6553600000, 13107199999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[13107200000, 26214399999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[26214400000, 52428799999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[52428800000, 104857599999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[104857600000, 209715199999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[209715200000, 419430399999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[419430400000, 838860799999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:37:24.536 INFO:tasks.workunit.client.0.smithi110.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "[53687091200000, None]" 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Request size (bytes)", 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 512, 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:24.537 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 511]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[512, 1023]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[1024, 2047]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[2048, 4095]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[4096, 8191]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[8192, 16383]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[16384, 32767]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[32768, 65535]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[65536, 131071]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[131072, 262143]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[262144, 524287]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[524288, 1048575]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[1048576, 2097151]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[2097152, 4194303]", 2024-09-14T03:37:24.538 INFO:tasks.workunit.client.0.smithi110.stdout: "[4194304, 8388607]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[8388608, 16777215]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[16777216, 33554431]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[33554432, 67108863]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[67108864, 134217727]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[134217728, 268435455]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[268435456, 536870911]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[536870912, 1073741823]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[1073741824, 2147483647]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[2147483648, 4294967295]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[4294967296, 8589934591]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[8589934592, 17179869183]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[17179869184, 34359738367]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[34359738368, 68719476735]", 2024-09-14T03:37:24.539 INFO:tasks.workunit.client.0.smithi110.stdout: "[68719476736, 137438953471]", 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: "[137438953472, 274877906943]", 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: "[274877906944, None]" 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: "osds": [ 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 0, 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.540 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.541 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.542 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 1, 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.543 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.544 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.545 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 2, 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.546 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.547 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.548 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 3, 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.549 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.550 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.551 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 4, 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.552 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.553 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.554 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 5, 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.555 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.556 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.557 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 6, 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.558 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.559 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 7, 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.560 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.561 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.562 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: "axes": [ 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:24.563 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Latency (usec)", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 100000, 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 99999]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[100000, 199999]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[200000, 399999]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[400000, 799999]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[800000, 1599999]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[1600000, 3199999]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[3200000, 6399999]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[6400000, 12799999]", 2024-09-14T03:37:24.564 INFO:tasks.workunit.client.0.smithi110.stdout: "[12800000, 25599999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[25600000, 51199999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[51200000, 102399999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[102400000, 204799999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[204800000, 409599999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[409600000, 819199999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[819200000, 1638399999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[1638400000, 3276799999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[3276800000, 6553599999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[6553600000, 13107199999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[13107200000, 26214399999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[26214400000, 52428799999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[52428800000, 104857599999]", 2024-09-14T03:37:24.565 INFO:tasks.workunit.client.0.smithi110.stdout: "[104857600000, 209715199999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[209715200000, 419430399999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[419430400000, 838860799999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "[53687091200000, None]" 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.566 INFO:tasks.workunit.client.0.smithi110.stdout: "buckets": 32, 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "min": 0, 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "name": "Request size (bytes)", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "quant_size": 512, 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "ranges": [ 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[None, -1]", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 511]", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[512, 1023]", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[1024, 2047]", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[2048, 4095]", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[4096, 8191]", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[8192, 16383]", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[16384, 32767]", 2024-09-14T03:37:24.567 INFO:tasks.workunit.client.0.smithi110.stdout: "[32768, 65535]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[65536, 131071]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[131072, 262143]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[262144, 524287]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[524288, 1048575]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[1048576, 2097151]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[2097152, 4194303]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[4194304, 8388607]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[8388608, 16777215]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[16777216, 33554431]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[33554432, 67108863]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[67108864, 134217727]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[134217728, 268435455]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[268435456, 536870911]", 2024-09-14T03:37:24.568 INFO:tasks.workunit.client.0.smithi110.stdout: "[536870912, 1073741823]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[1073741824, 2147483647]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[2147483648, 4294967295]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[4294967296, 8589934591]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[8589934592, 17179869183]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[17179869184, 34359738367]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[34359738368, 68719476735]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[68719476736, 137438953471]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[137438953472, 274877906943]", 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "[274877906944, None]" 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: "scale_type": "log2" 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.569 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "osds": [ 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 0, 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.570 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.571 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.572 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 1, 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 6, 3, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 13, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.573 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.574 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.575 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 2, 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.576 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.577 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 3, 2024-09-14T03:37:24.578 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.579 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.580 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 4, 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.581 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.582 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.583 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 5, 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.584 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.585 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.586 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 6, 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.587 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.588 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.589 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "osd_id": 7, 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "values": [ 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.590 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.591 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:37:24.592 INFO:tasks.workunit.client.0.smithi110.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: ] 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: "perf_counters": { 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: "mds.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: "mds": { 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: "caps": { 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.593 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_flush_ack": { 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_grant": { 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_revoke": { 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "ceph_cap_op_trunc": { 2024-09-14T03:37:24.594 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "dir_commit": { 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "dir_fetch": { 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "dir_merge": { 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "dir_split": { 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.595 INFO:tasks.workunit.client.0.smithi110.stdout: "exported_inodes": { 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "forward": { 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_cap_release": { 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_caps": { 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_caps_dirty": { 2024-09-14T03:37:24.596 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_inode_file_caps": { 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: "imported_inodes": { 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: "inodes": { 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: "inodes_expired": { 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.597 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "inodes_pinned": { 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "inodes_with_caps": { 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "load_cent": { 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "openino_dir_fetch": { 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "process_request_cap_release": { 2024-09-14T03:37:24.598 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "reply_latency": { 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 150712 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "request": { 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "root_rbytes": { 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "root_rfiles": { 2024-09-14T03:37:24.599 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "root_rsnaps": { 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_reply": { 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "subtrees": { 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_cache": { 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_enqueue_scrub": { 2024-09-14T03:37:24.600 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_exportdir": { 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_flush": { 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_fragmentdir": { 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_fragstats": { 2024-09-14T03:37:24.601 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: "ireq_inodestats": { 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_recovering_enqueued": { 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_recovering_prioritized": { 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: "num_recovering_processing": { 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.602 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_strays": { 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_strays_delayed": { 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_strays_enqueuing": { 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_completed": { 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.603 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_started": { 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "strays_created": { 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "strays_enqueued": { 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "strays_migrated": { 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "strays_reintegrated": { 2024-09-14T03:37:24.604 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_log": { 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: "ev": { 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: "evadd": { 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: "evex": { 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.605 INFO:tasks.workunit.client.0.smithi110.stdout: "evexd": { 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "evexg": { 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "evtrm": { 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "jlat": { 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11187690 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "replayed": { 2024-09-14T03:37:24.606 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: "seg": { 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: "segadd": { 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: "segex": { 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: "segexd": { 2024-09-14T03:37:24.607 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: "segexg": { 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: "segtrm": { 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_mem": { 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: "cap": { 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.608 INFO:tasks.workunit.client.0.smithi110.stdout: "cap+": { 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "cap-": { 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "dir": { 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "dir+": { 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "dir-": { 2024-09-14T03:37:24.609 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "dn": { 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "dn+": { 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "dn-": { 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "heap": { 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 207068 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.610 INFO:tasks.workunit.client.0.smithi110.stdout: "ino": { 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "ino+": { 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "ino-": { 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "rss": { 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 37880 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_server": { 2024-09-14T03:37:24.611 INFO:tasks.workunit.client.0.smithi110.stdout: "cap_acquisition_throttle": { 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "cap_revoke_eviction": { 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_request": { 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_client_session": { 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 16 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "handle_peer_request": { 2024-09-14T03:37:24.612 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "req_create_latency": { 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "req_getattr_latency": { 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 150712 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "req_getfilelock_latency": { 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.613 INFO:tasks.workunit.client.0.smithi110.stdout: "req_getvxattr_latency": { 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "req_link_latency": { 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookup_latency": { 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.614 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookuphash_latency": { 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookupino_latency": { 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookupname_latency": { 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookupparent_latency": { 2024-09-14T03:37:24.615 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lookupsnap_latency": { 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "req_lssnap_latency": { 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "req_mkdir_latency": { 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.616 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "req_mknod_latency": { 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "req_mksnap_latency": { 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "req_open_latency": { 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.617 INFO:tasks.workunit.client.0.smithi110.stdout: "req_readdir_latency": { 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "req_rename_latency": { 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "req_renamesnap_latency": { 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "req_rmdir_latency": { 2024-09-14T03:37:24.618 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "req_rmsnap_latency": { 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "req_rmxattr_latency": { 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setattr_latency": { 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.619 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setdirlayout_latency": { 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setfilelock_latency": { 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setlayout_latency": { 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.620 INFO:tasks.workunit.client.0.smithi110.stdout: "req_setxattr_latency": { 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "req_symlink_latency": { 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "req_unlink_latency": { 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.621 INFO:tasks.workunit.client.0.smithi110.stdout: "mds_sessions": { 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "average_load": { 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "avg_session_uptime": { 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 87 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "mdthresh_evicted": { 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "session_add": { 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.622 INFO:tasks.workunit.client.0.smithi110.stdout: "session_count": { 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "session_remove": { 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "sessions_open": { 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "sessions_stale": { 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "total_load": { 2024-09-14T03:37:24.623 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.624 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: "purge_queue": { 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executed": { 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executing": { 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executing_high_water": { 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.625 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executing_ops": { 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_executing_ops_high_water": { 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: "pq_item_in_journal": { 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.1568173bdc0b4a394924040a7d808e7f81e54181": { 2024-09-14T03:37:24.626 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "election_call": { 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "election_lose": { 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "election_win": { 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_elections": { 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": { 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: "session_add": { 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: "session_rm": { 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 185 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: "session_trim": { 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.628 INFO:tasks.workunit.client.0.smithi110.stdout: "paxos": { 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "accept_timeout": { 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "begin": { 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 933 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_bytes": { 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5015259 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_keys": { 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:24.629 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2799 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_latency": { 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 135997710 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "collect": { 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_bytes": { 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.630 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_keys": { 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_latency": { 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_timeout": { 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_uncommitted": { 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.631 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "commit": { 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 933 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_bytes": { 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4948141 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_keys": { 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5347 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.632 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_latency": { 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 933, 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 125875915 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_ack_timeout": { 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_timeout": { 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn": { 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.633 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn_latency": { 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh": { 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 936 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh_latency": { 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 936, 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2195963021 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "restart": { 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.634 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state": { 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_bytes": { 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 473 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_keys": { 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "start_leader": { 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.635 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "start_peon": { 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state": { 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_bytes": { 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_keys": { 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.636 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_latency": { 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache": { 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1020054731 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 46956544 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.637 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 42328064 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2147483648 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4628480 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:full": { 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:24.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277021587 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 273125 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.639 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.640 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 70832520 2024-09-14T03:37:24.641 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:inc": { 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277021587 2024-09-14T03:37:24.642 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 39004 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.643 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.644 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71066641 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:kv": { 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 322961408 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.645 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 184681058 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.646 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.647 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71171486 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb": { 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "compact": { 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_completed": { 2024-09-14T03:37:24.648 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lasted": { 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 31253947 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_len": { 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_merge": { 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_running": { 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.649 INFO:tasks.workunit.client.0.smithi110.stdout: "get_latency": { 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 41572, 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 149350771 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:37:24.650 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_latency": { 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.651 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_sync_latency": { 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2030, 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 264315066 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.3ecc698db0ba77df02613a93e9949ac11557d553": { 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "election_call": { 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "election_lose": { 2024-09-14T03:37:24.652 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: "election_win": { 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_elections": { 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": { 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: "session_add": { 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "session_rm": { 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 50 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "session_trim": { 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "paxos": { 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "accept_timeout": { 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "begin": { 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 812 2024-09-14T03:37:24.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_bytes": { 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 812, 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3974355 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_keys": { 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_latency": { 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 812, 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 107997942 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.655 INFO:tasks.workunit.client.0.smithi110.stdout: "collect": { 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_bytes": { 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 219 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_keys": { 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_latency": { 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.656 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_timeout": { 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_uncommitted": { 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "commit": { 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 813 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_bytes": { 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.657 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_keys": { 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_latency": { 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_ack_timeout": { 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_timeout": { 2024-09-14T03:37:24.658 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn": { 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn_latency": { 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh": { 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 813 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh_latency": { 2024-09-14T03:37:24.659 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 813, 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1637968651 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "restart": { 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state": { 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_bytes": { 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.660 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_keys": { 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "start_leader": { 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "start_peon": { 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state": { 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 813 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.661 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_bytes": { 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 813, 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7796231 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_keys": { 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 813, 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5498 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_latency": { 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 813, 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 118997720 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache": { 2024-09-14T03:37:24.662 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1020054731 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44892160 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 34603008 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2147483648 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.663 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10289152 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:full": { 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.664 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277020525 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 273555 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.665 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.666 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 70833152 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:inc": { 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.667 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277020525 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 41407 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.668 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.669 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71065300 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.670 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:kv": { 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 322961408 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 184680350 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.671 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.672 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.673 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71172194 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb": { 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "compact": { 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_completed": { 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lasted": { 2024-09-14T03:37:24.674 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 20276919 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_len": { 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_merge": { 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_running": { 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "get_latency": { 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 35085, 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 79047830 2024-09-14T03:37:24.675 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.676 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_latency": { 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_sync_latency": { 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1793, 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 255983165 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.677 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "mon.b030e701d31b8bd7b2036a36a31490ca893f182a": { 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "election_call": { 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "election_lose": { 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "election_win": { 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "num_elections": { 2024-09-14T03:37:24.678 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: "num_sessions": { 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: "session_add": { 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: "session_rm": { 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 78 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: "session_trim": { 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.679 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "paxos": { 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "accept_timeout": { 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "begin": { 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 787 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_bytes": { 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3899829 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_keys": { 2024-09-14T03:37:24.680 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "begin_latency": { 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 86998539 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "collect": { 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_bytes": { 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73 2024-09-14T03:37:24.681 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_keys": { 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_latency": { 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 999984 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_timeout": { 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "collect_uncommitted": { 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.682 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "commit": { 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 787 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_bytes": { 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_keys": { 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "commit_latency": { 2024-09-14T03:37:24.683 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_ack_timeout": { 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "lease_timeout": { 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn": { 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "new_pn_latency": { 2024-09-14T03:37:24.684 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh": { 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 787 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "refresh_latency": { 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1769970170 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "restart": { 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state": { 2024-09-14T03:37:24.685 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_bytes": { 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "share_state_keys": { 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "start_leader": { 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.686 INFO:tasks.workunit.client.0.smithi110.stdout: "start_peon": { 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state": { 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 787 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_bytes": { 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7650883 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_keys": { 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5312 2024-09-14T03:37:24.687 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "store_state_latency": { 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 787, 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 90998468 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache": { 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1020054731 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 43835392 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.688 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 32112640 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2147483648 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11722752 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:full": { 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.689 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277020525 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 273555 2024-09-14T03:37:24.690 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.691 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 70833152 2024-09-14T03:37:24.692 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:inc": { 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 348127232 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 277020525 2024-09-14T03:37:24.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 41407 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.694 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.695 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71065300 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "prioritycache:kv": { 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 322961408 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.696 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 184680350 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.697 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.698 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71172194 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb": { 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: "compact": { 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.699 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_completed": { 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lasted": { 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 30515755 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_len": { 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_queue_merge": { 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_running": { 2024-09-14T03:37:24.700 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "get_latency": { 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 33598, 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 80203642 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.701 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_latency": { 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "submit_sync_latency": { 2024-09-14T03:37:24.702 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1739, 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 202908699 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.0": { 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:24.703 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:24.704 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2420736 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:24.705 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1630208 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1490944 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.706 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:24.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:24.709 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.710 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.711 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3985408 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.712 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1306624 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270560 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.713 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 19, 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56215 2024-09-14T03:37:24.714 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 347, 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 59208500 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 337, 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10853986 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:24.715 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 347, 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 375197 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 347, 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 59583697 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.716 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 30231 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10095 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.717 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11661 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 721 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 224 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 115, 2024-09-14T03:37:24.718 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1478999 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 276, 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 248347 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 161, 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1006797 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:24.719 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 49262 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5356489 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 101, 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6737607 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 101, 2024-09-14T03:37:24.720 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 494004958405 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 101, 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 135868615904 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 57190608630 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 51155 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.721 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 405567 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 122807626 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 139827 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.722 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 61621572 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 30093122 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 903228 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 61, 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 274973 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:24.723 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 220343525 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 35818606 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 437, 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2880808 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:24.724 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861437685 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270974976 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270721024 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137345945 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.725 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 253952 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.726 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85496386 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.727 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.728 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73883070 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.729 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641222901 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.730 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:24.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71806651 2024-09-14T03:37:24.732 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.733 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56997591 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.734 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.735 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68831065 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.736 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641222901 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9926 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.737 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 273675 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.738 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71525178 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.739 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45 2024-09-14T03:37:24.740 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:24.741 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 604 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 6, 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44652978 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 7, 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3427038 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:24.742 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 6, 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 43098701 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.743 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.744 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.745 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.746 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 604 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 6, 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44652978 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:24.747 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 7, 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3427038 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 6, 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 43098701 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.748 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305795072 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.1": { 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.749 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.750 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2064384 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.751 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1413120 2024-09-14T03:37:24.752 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:24.754 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:24.755 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:24.756 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.757 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.758 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3411968 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 696320 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:24.759 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 210139 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.760 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 26, 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 79973 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56417516 2024-09-14T03:37:24.761 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 282, 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10370644 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 386709 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56804225 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.762 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 31535 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9184 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:24.763 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18983 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 660 2024-09-14T03:37:24.764 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 227 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 116, 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1991654 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 272, 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 246039 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 156, 2024-09-14T03:37:24.765 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1318838 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 106809 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5502311 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.766 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 96, 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6718397 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 95, 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 346766756657 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 96, 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 105748426944 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:24.767 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 348, 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 27823266923 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 348, 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 36602 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 351053 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.768 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 198969530 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 109650 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 59104874 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18677552 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.769 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 311378 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 55, 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 318006 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 282653249 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.770 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 22579974 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 349, 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2250577 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861437685 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269860864 2024-09-14T03:37:24.771 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269443072 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137345945 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 417792 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.772 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85497569 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.773 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.774 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73873695 2024-09-14T03:37:24.775 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.776 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641231768 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.777 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.778 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71797784 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.779 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998379 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.780 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.781 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68830277 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.782 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641231768 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.783 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.784 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 252908 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.785 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71547004 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.786 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 37 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 97 2024-09-14T03:37:24.787 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1640827 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 97, 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 242941886 2024-09-14T03:37:24.788 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 608 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 97, 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 33927786 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 97, 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 120526059 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.789 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 18, 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2595833 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 608 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 18, 2024-09-14T03:37:24.790 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2313159 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 18, 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1888538 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:24.791 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9485040 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:24.792 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4654053 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8901335 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1640827 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:24.793 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 67, 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 230861013 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 67, 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 26960574 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 67, 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 109736186 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.794 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 590368 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:24.795 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305188864 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.2": { 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:24.796 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.797 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1769472 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.798 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1200128 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1060864 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.799 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.800 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:24.801 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.802 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.803 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:24.804 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2904064 2024-09-14T03:37:24.805 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 696320 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 239697 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:24.806 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:24.807 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 19696 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 242, 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 38437610 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 239, 2024-09-14T03:37:24.808 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7391541 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 242, 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 365978 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 242, 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 38803588 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.809 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 36254 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1680 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:24.810 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5672 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 513 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 208 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:24.811 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 96, 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1489412 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 227, 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 225297 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 131, 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 992963 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.812 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85488 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5326499 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 90, 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5724775 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.813 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 89, 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 313317598528 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 90, 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 130779151433 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 289, 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 64949844443 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 289, 2024-09-14T03:37:24.814 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 29490 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 335042 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67983316 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95019 2024-09-14T03:37:24.815 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 30687428 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 16404673 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312144 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 50, 2024-09-14T03:37:24.816 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 262564 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 120777644 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 19715667 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 290, 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2311227 2024-09-14T03:37:24.817 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861437685 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270139392 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269410304 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.818 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137345945 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 729088 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.819 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85497502 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.820 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 20480 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.821 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73865570 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.822 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641231265 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.823 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:24.824 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71798287 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.825 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998334 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.826 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.827 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.828 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68830322 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.829 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641231265 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.830 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 245836 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.831 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71554579 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.832 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 31 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.833 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.834 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 108307392 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4455016 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 28163644 2024-09-14T03:37:24.835 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:24.836 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:24.837 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:24.838 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:24.839 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 108307392 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4455016 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.840 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 10, 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 28163644 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.841 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305188864 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.3": { 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:24.842 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.843 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1142784 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.844 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 897024 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 757760 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.845 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:24.846 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.847 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:24.848 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:24.849 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.850 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:24.851 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1974272 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 716800 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.852 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.853 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6719 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 168, 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 29425379 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:24.854 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 156, 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11524890 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 168, 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 211626 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 168, 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 29637005 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.855 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 36962 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 833 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.856 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8126 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 316 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 222 2024-09-14T03:37:24.857 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 84, 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 763951 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 140, 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 287256 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 56, 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 216863 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:24.858 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4073079 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 45, 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2984112 2024-09-14T03:37:24.859 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 42, 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 326857821236 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 45, 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 66141850537 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 230, 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 509494 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.860 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 230, 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23501 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 336837 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 91660724 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.861 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65953 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 43687634 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11172381 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 314872 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:24.862 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 36, 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 190044 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 150958930 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13643909 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 233, 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1639711 2024-09-14T03:37:24.863 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861437685 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 268877824 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 268681216 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.864 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137345945 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 196608 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.865 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85498046 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.866 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.867 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73881410 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.868 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641235350 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.869 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.870 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71794202 2024-09-14T03:37:24.871 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.872 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998697 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.873 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.874 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68829959 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.875 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641235350 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.876 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6954 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 248981 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.877 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71540395 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:24.878 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:24.879 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 36 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.880 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 720598 2024-09-14T03:37:24.881 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.882 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:24.883 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.885 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.886 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 720598 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.887 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305205248 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.4": { 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.888 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:24.889 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1302528 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.890 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1003520 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 864256 2024-09-14T03:37:24.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.892 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.893 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:24.894 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.895 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:24.896 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2240512 2024-09-14T03:37:24.897 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 716800 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 341939 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:24.898 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:24.899 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 11, 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 32115 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 194, 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45336484 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 186, 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6545617 2024-09-14T03:37:24.900 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 194, 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 220758 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 194, 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45557242 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.901 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 34110 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6665 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.902 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9762 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 371 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 235 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:24.903 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 92, 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1179161 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 153, 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 214272 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 61, 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 533475 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.904 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 104651 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4464750 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 41, 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2586832 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 40, 2024-09-14T03:37:24.905 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 205514579974 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 41, 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 61296360286 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 263, 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8307274851 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 263, 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 46187 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.906 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 241652 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 101423670 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 150543 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:24.907 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 58578829 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12659084 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312859 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 39, 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 168919 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.908 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 177403608 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15337113 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 264, 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2149014 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.909 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861443778 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270139392 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269836288 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137353113 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.910 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 303104 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.911 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85498317 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.912 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.913 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73885235 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641237377 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.915 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.916 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71792175 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.917 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998878 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.918 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.919 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68829778 2024-09-14T03:37:24.920 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641237377 2024-09-14T03:37:24.921 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.922 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 256053 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.923 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71538250 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 39 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 22 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7100463 2024-09-14T03:37:24.926 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 618950 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7026259 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.927 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:24.928 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.929 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.930 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:24.931 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 22 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7100463 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 618950 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:24.932 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7026259 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:24.933 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305209344 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.5": { 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:24.934 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1101824 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.936 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 811008 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 671744 2024-09-14T03:37:24.937 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:24.938 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:24.939 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:24.940 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.941 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:24.942 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.943 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1847296 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 708608 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 4, 2024-09-14T03:37:24.944 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 137847 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:24.945 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18987 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 147, 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 45446094 2024-09-14T03:37:24.946 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 145, 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3995658 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 147, 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 724148 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 147, 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 46170242 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.947 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 32122 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4503 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.948 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4066 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 335 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.949 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 242 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 86, 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 818237 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 135, 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 264641 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 49, 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 350694 2024-09-14T03:37:24.950 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 77891 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2788885 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:24.951 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 40, 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2849106 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 37, 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270364427588 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 40, 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 58289044221 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 267, 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8140157513 2024-09-14T03:37:24.952 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 267, 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 27585 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 222640 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 111535458 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.953 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 96261 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 145447605 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12680853 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 314318 2024-09-14T03:37:24.954 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 32, 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 149299 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 272732474 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14697154 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:24.955 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 270, 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1968038 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861443778 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270974976 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270098432 2024-09-14T03:37:24.956 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137353113 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 876544 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.957 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85498154 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.958 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.959 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73885398 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:24.960 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641236156 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.961 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:24.962 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71793396 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.963 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998769 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.964 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.965 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.966 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68829887 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.967 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641236156 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:24.968 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6956 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 257377 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.969 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71531191 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.970 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 42 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.971 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.972 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.973 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.974 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.975 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.976 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.977 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:24.978 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 590368 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.979 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305197056 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.6": { 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.980 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.981 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1220608 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.982 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 942080 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 802816 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.983 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:24.984 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.985 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.986 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.987 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:24.988 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2097152 2024-09-14T03:37:24.989 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1298432 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:24.990 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:24.991 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 8, 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 27563 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 179, 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 40751332 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 168, 2024-09-14T03:37:24.992 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9034851 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 179, 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 156284 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 179, 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 40907616 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.993 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 39098 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4627 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.994 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 10553 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 347 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 244 2024-09-14T03:37:24.995 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 92, 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 832669 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 145, 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 215531 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 53, 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 392999 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.996 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4888241 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 35, 2024-09-14T03:37:24.997 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2554457 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 35, 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 440087844585 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 35, 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 48409185137 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 439957 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.998 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 28020 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 828161 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 98400919 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:24.999 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 80010 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 60646315 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11572823 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 904821 2024-09-14T03:37:25.000 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 28, 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 122883 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 176373022 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 16305052 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.001 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 259, 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2159421 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861443778 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 271187968 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270016512 2024-09-14T03:37:25.002 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137353113 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1171456 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:25.003 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85498092 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.004 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:25.005 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73885460 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:25.006 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641235693 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.007 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:25.008 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71793859 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.009 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56998728 2024-09-14T03:37:25.010 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:25.011 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.012 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68829928 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.013 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641235693 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.014 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7095 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 258267 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.015 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71530625 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:25.016 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 39 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.017 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2804 2024-09-14T03:37:25.018 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 13, 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68458525 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 15, 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6971829 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.019 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 13, 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 38597408 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.020 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:25.021 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:25.022 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.023 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2804 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 13, 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68458525 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:25.024 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 15, 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 6971829 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 13, 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 38597408 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 590368 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.025 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305786880 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: "osd.7": { 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: "bluefs": { 2024-09-14T03:37:25.026 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_fallback": { 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_db": { 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 65536 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_main": { 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit_wal": { 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.027 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_slow": { 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_sst": { 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 12288 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_written_wal": { 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1228800 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lat": { 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.028 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "compact_lock_lat": { 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "db_total_bytes": { 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "db_used_bytes": { 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 304480256 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "log_bytes": { 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 958464 2024-09-14T03:37:25.029 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "logged_bytes": { 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 819200 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_db": { 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 312868864 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_slow": { 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "max_bytes_wal": { 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "num_files": { 2024-09-14T03:37:25.030 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: "read_bytes": { 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 195697 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: "read_count": { 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 23 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes": { 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1269760 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_db": { 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1273856 2024-09-14T03:37:25.031 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_slow": { 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_bytes_wal": { 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "read_disk_count": { 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 15 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_bytes": { 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4789 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "read_prefetch_count": { 2024-09-14T03:37:25.032 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_bytes": { 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_buffer_count": { 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_bytes": { 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5589 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_count": { 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 24 2024-09-14T03:37:25.033 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes": { 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "read_random_disk_count": { 2024-09-14T03:37:25.034 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_total_bytes": { 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: "slow_used_bytes": { 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_total_bytes": { 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: "wal_used_bytes": { 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.035 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "write_bytes": { 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2121728 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore": { 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "alloc_unit": { 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 4096 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": { 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1298432 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "clist_lat": { 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 3, 2024-09-14T03:37:25.036 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 137018 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_lat": { 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed": { 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_allocated": { 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.037 INFO:tasks.workunit.client.0.smithi110.stdout: "compressed_original": { 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "csum_lat": { 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 12, 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 29349 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "decompress_lat": { 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_commit_lat": { 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 183, 2024-09-14T03:37:25.038 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 52582473 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_final_lat": { 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 170, 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 13584141 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_flush_lat": { 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 183, 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 133383 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "kv_sync_lat": { 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 183, 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 52715856 2024-09-14T03:37:25.039 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_keys_lat": { 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_get_values_lat": { 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44957 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_lower_bound_lat": { 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1727 2024-09-14T03:37:25.040 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_next_lat": { 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 0, 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_upper_bound_lat": { 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 2, 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 9046 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.041 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_hits": { 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 549 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "onode_misses": { 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 268 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "read_lat": { 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 114, 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1218152 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "read_onode_meta_lat": { 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 181, 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 213615 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.042 INFO:tasks.workunit.client.0.smithi110.stdout: "read_wait_aio_lat": { 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 67, 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 600635 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "reads_with_retries": { 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "remove_lat": { 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 1, 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 17356 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "state_aio_wait_lat": { 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.043 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3714527 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 24, 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1670581 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 24, 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 274136803677 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "state_deferred_queued_lat": { 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 24, 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 34902155110 2024-09-14T03:37:25.044 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "state_done_lat": { 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 436083 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "state_finishing_lat": { 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 41083 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "state_io_done_lat": { 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1418046 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_commiting_lat": { 2024-09-14T03:37:25.045 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 179425563 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_done_lat": { 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 101678 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "state_kv_queued_lat": { 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 120748548 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "state_prepare_lat": { 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.046 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 11964813 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": { 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 903104 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "truncate_lat": { 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 23, 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 97298 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_commit_lat": { 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 317328000 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_submit_lat": { 2024-09-14T03:37:25.047 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 14767434 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "txc_throttle_lat": { 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 294, 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1973643 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache": { 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_bytes": { 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1861443778 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "heap_bytes": { 2024-09-14T03:37:25.048 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 270974976 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "mapped_bytes": { 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 269631488 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "target_bytes": { 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3137353113 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "unmapped_bytes": { 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 1343488 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:data": { 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:25.049 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 159383552 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 85488841 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.050 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:25.051 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 139264 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 73755447 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.052 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv": { 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 780140544 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641166308 2024-09-14T03:37:25.053 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.054 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2128 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:25.055 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71863244 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 192937984 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.056 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 67108864 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 464 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 56992560 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:25.057 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.058 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 68836096 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.059 INFO:tasks.workunit.client.0.smithi110.stdout: "bluestore-pricache:meta": { 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "committed_bytes": { 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 713031680 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "pri0_bytes": { 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "pri10_bytes": { 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "pri11_bytes": { 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 641166308 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.060 INFO:tasks.workunit.client.0.smithi110.stdout: "pri1_bytes": { 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "pri2_bytes": { 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "pri3_bytes": { 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "pri4_bytes": { 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "pri5_bytes": { 2024-09-14T03:37:25.061 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "pri6_bytes": { 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 274596 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "pri7_bytes": { 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "pri8_bytes": { 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "pri9_bytes": { 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.062 INFO:tasks.workunit.client.0.smithi110.stdout: "reserved_bytes": { 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71590776 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: "objecter": { 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: "op_active": { 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rmw": { 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.063 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "osd": { 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg": { 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 46 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "numpg_removing": { 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "op": { 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 87 2024-09-14T03:37:25.064 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_degraded": { 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "op_delayed_unreadable": { 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "op_in_bytes": { 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "op_latency": { 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 87, 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 71925275 2024-09-14T03:37:25.065 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "op_out_bytes": { 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 264333 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "op_prepare_latency": { 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 89, 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 19822920 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "op_process_latency": { 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 87, 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 50673872 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r": { 2024-09-14T03:37:25.066 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 60 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency": { 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 60, 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8251032 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_out_bytes": { 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 264333 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_prepare_latency": { 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 60, 2024-09-14T03:37:25.067 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 7307006 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "op_r_process_latency": { 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 60, 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5852018 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw": { 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 22 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_in_bytes": { 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency": { 2024-09-14T03:37:25.068 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 22, 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 19621401 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_out_bytes": { 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_prepare_latency": { 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 22, 2024-09-14T03:37:25.069 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 8596498 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "op_rw_process_latency": { 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 22, 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 18279099 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w": { 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 5 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_in_bytes": { 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency": { 2024-09-14T03:37:25.070 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 44052842 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_prepare_latency": { 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 7, 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 3919416 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "op_w_process_latency": { 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "count": 5, 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 26542755 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.071 INFO:tasks.workunit.client.0.smithi110.stdout: "op_wip": { 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 0 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_bytes": { 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 590368 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "recovery_ops": { 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 2 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes": { 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 95995035648 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_bytes_used": { 2024-09-14T03:37:25.072 INFO:tasks.workunit.client.0.smithi110.stdout: "value": 305786880 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: "pools": [ 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr" 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:25.073 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 1, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 1, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 1, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115901194, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 1781760, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 1781760, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:25.074 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 2, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 2.450442480039783e-06, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 272, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 490496, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 590368, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 590368, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 1771104, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 145, 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 1667072 2024-09-14T03:37:25.075 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:25.076 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 32, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 32, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 2, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115901194, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 98304, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 98304, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:25.077 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 22, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 3410, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 3410, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 10230, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 51, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 19456 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.078 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 64, 2024-09-14T03:37:25.079 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 64, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 3, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115901194, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 0, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 0, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 0, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:25.080 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 0.0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 0 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:25.081 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd" 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "cache_mode": "none", 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "erasure_code_profile": {}, 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "min_size": 2, 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "options": {}, 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 8, 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pgp_num": 8, 2024-09-14T03:37:25.082 INFO:tasks.workunit.client.0.smithi110.stdout: "pool": 4, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "size": 3, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "stats": { 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_raw": 727115901194, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "bytes_used": 12288, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_bytes_used": 0, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "compress_under_bytes": 0, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "data_bytes_used": 12288, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty": 0, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "max_avail": 242371969024, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "objects": 2, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_bytes_used": 0, 2024-09-14T03:37:25.083 INFO:tasks.workunit.client.0.smithi110.stdout: "percent_used": 1.689964435058755e-08, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_bytes": 0, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "quota_objects": 0, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "rd": 0, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "rd_bytes": 0, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "stored": 19, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_data": 19, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_omap": 0, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "stored_raw": 57, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "wr": 2, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "wr_bytes": 2048 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_bytes": 0, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "target_max_objects": 0, 2024-09-14T03:37:25.084 INFO:tasks.workunit.client.0.smithi110.stdout: "type": "replicated" 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd": { 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: "mirroring_by_pool": [ 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: false 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: "num_images_by_pool": [ 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pools": 1 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.085 INFO:tasks.workunit.client.0.smithi110.stdout: "report_id": "2dc8c2a9-6f51-4b3c-b455-44a7525f06ca", 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "report_timestamp": "2024-09-14T03:37:23.552715", 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "report_version": 1, 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "rocksdb_stats": { 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "6.15.5" 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "rook": { 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "cluster": { 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "external-mode": null, 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "mon": { 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "allow-multiple-per-node": null, 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "count": null, 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "max-id": null, 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "pvc": { 2024-09-14T03:37:25.086 INFO:tasks.workunit.client.0.smithi110.stdout: "enabled": null 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "stretch": { 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "enabled": null 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "network": { 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "provider": null 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "storage": { 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "device-set": { 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "non-portable": null, 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "portable": null, 2024-09-14T03:37:25.087 INFO:tasks.workunit.client.0.smithi110.stdout: "total": null 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: "csi": { 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: "kubernetes": { 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: "node": { 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:25.088 INFO:tasks.workunit.client.0.smithi110.stdout: "kubernetes-total": null, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "with-ceph-daemons": null, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-cephfs-plugin": null, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-nfs-plugin": null, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "with-csi-rbd-plugin": null 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "usage": { 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "storage-class": { 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "count": { 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "bucket": null, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs": null, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "nfs": null, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd": null, 2024-09-14T03:37:25.089 INFO:tasks.workunit.client.0.smithi110.stdout: "total": null 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: "version": null 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: "services": {}, 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_per_pg": [ 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.090 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.518530+0000", 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.518530+0000", 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.519367+0000", 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:25.091 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916845+0000", 2024-09-14T03:37:25.092 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.093 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3f", 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:02:36.205002+0000", 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.094 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.095 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.096 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.097 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.098 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.514951+0000", 2024-09-14T03:37:25.099 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.514951+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515212+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:25.100 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837707+0000", 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.101 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3e", 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:08:30.742131+0000", 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.102 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.103 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.104 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.105 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.106 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.107 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.107 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.107 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.107 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.107 INFO:tasks.workunit.client.0.smithi110.stderr:+ echo OK 2024-09-14T03:37:25.108 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.111 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.112 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:23 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.116 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.116 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.116 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:25.116 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.504550+0000", 2024-09-14T03:37:25.116 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.504550+0000", 2024-09-14T03:37:25.116 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.504777+0000", 2024-09-14T03:37:25.116 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831163+0000", 2024-09-14T03:37:25.117 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.118 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3d", 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:17:59.864492+0000", 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.119 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.120 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.121 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.122 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.123 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.521149+0000", 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.521149+0000", 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521444+0000", 2024-09-14T03:37:25.124 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839205+0000", 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.125 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3c", 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.126 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:27:10.004625+0000", 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.127 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.128 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.129 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.130 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.131 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512580+0000", 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512580+0000", 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513133+0000", 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.132 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916561+0000", 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.133 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3b", 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.134 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:43:03.077106+0000", 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.135 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.136 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.137 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.138 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:25.139 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.507921+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.507921+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.508230+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.140 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839116+0000", 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.141 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3a", 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:59:37.418870+0000", 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.142 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.143 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.144 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.145 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.146 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.515022+0000", 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.515022+0000", 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515319+0000", 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:25.147 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.915849+0000", 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.148 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.39", 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.149 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:51:51.089278+0000", 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.150 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.151 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.152 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.153 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.154 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.522063+0000", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.522063+0000", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.522349+0000", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:25.155 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859702+0000", 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.156 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.38", 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:16:15.378220+0000", 2024-09-14T03:37:25.157 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.158 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.159 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.160 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.161 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.162 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511364+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511364+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512031+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:25.163 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917135+0000", 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.164 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.37", 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:54:15.711934+0000", 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.165 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.166 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.167 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.168 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.169 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:25.170 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.518575+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.518575+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.518932+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.171 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917254+0000", 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.172 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.36", 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:01:11.929183+0000", 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.173 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.174 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.175 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.176 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.177 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.517385+0000", 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.517385+0000", 2024-09-14T03:37:25.178 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517977+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917339+0000", 2024-09-14T03:37:25.179 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.180 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.35", 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:42:50.900533+0000", 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.181 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.182 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.183 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.184 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.185 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.510519+0000", 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.510519+0000", 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.510949+0000", 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.186 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830573+0000", 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.187 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.34", 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.188 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:06:16.860908+0000", 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.189 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.190 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.191 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:25.192 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.193 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512201+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512201+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512463+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:25.194 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859767+0000", 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.195 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.33", 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:04:46.716482+0000", 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.196 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.197 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.198 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.199 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.200 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512773+0000", 2024-09-14T03:37:25.201 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512773+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513215+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:25.202 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830640+0000", 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.203 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.32", 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:27:04.454480+0000", 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.204 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.205 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.206 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.207 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.208 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.503212+0000", 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.503212+0000", 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.503785+0000", 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.209 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.836868+0000", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.210 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.31", 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.211 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:51:15.408242+0000", 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.212 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.213 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.214 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.215 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.216 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.953477+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.953477+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.953866+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:25.217 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.838932+0000", 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.218 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.30", 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 31, 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:56:22.262447+0000", 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.219 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.220 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.221 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.222 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.223 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.506907+0000", 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.506907+0000", 2024-09-14T03:37:25.224 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.507186+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839661+0000", 2024-09-14T03:37:25.225 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2f", 2024-09-14T03:37:25.226 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:10:47.570717+0000", 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.227 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.228 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.229 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.230 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.231 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.518795+0000", 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.518795+0000", 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.519104+0000", 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.232 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830684+0000", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.233 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2e", 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.234 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:34:49.356603+0000", 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.235 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.236 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.237 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.238 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.239 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.521559+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.521559+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521848+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.240 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916926+0000", 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.241 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2d", 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:21:09.742073+0000", 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.242 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.243 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.244 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.245 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.246 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.514291+0000", 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.514291+0000", 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515094+0000", 2024-09-14T03:37:25.247 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916978+0000", 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.248 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2c", 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.249 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:39:52.842240+0000", 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.250 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.251 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.252 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.253 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.254 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.510485+0000", 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.510485+0000", 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.510758+0000", 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.255 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831065+0000", 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.256 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2b", 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.257 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:26:25.102759+0000", 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.258 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.259 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.260 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.261 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.262 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511289+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511289+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.511696+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.263 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839132+0000", 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.264 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2a", 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:02:27.657439+0000", 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.265 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.266 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.267 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.268 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.269 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.517314+0000", 2024-09-14T03:37:25.270 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.517314+0000", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517564+0000", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.271 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859569+0000", 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.272 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.29", 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:55:08.360046+0000", 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.273 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.274 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.275 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.276 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.277 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.513008+0000", 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.513008+0000", 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513865+0000", 2024-09-14T03:37:25.278 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.838946+0000", 2024-09-14T03:37:25.279 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.280 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.28", 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:18:37.581596+0000", 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.281 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.282 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.283 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.284 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.285 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.517252+0000", 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.517252+0000", 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517481+0000", 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.286 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859955+0000", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.287 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.27", 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.288 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:29:15.209238+0000", 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.289 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.290 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.291 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.292 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.293 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.520764+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.520764+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521201+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:25.294 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917253+0000", 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.295 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.26", 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:39:40.688146+0000", 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.296 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.297 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.298 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.299 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.300 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:25.301 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.503906+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.503906+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.504156+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.302 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837307+0000", 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.303 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.25", 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:33:57.350618+0000", 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.304 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.305 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.306 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.307 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.308 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.330493+0000", 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.330493+0000", 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.330855+0000", 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.309 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859822+0000", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.310 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.24", 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.311 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 31, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:28:06.278074+0000", 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.312 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.313 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.314 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.315 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.316 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.955095+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.955095+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.955446+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:25.317 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830387+0000", 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.318 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.23", 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 18, 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:56:01.417881+0000", 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.319 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.320 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.321 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.322 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.323 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512998+0000", 2024-09-14T03:37:25.324 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512998+0000", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513294+0000", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.325 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.328 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:25.328 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839234+0000", 2024-09-14T03:37:25.328 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.328 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.328 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.328 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.328 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.328 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.22", 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.329 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:04:54.262830+0000", 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.330 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.331 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.332 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.333 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.334 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511955+0000", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511955+0000", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512389+0000", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:25.335 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830045+0000", 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.336 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.21", 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:30:31.454464+0000", 2024-09-14T03:37:25.337 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.338 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.339 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.340 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.341 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519764+0000", 2024-09-14T03:37:25.342 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519764+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520047+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:25.343 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.915824+0000", 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.344 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.20", 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:47:55.233617+0000", 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.345 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.346 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.347 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.348 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.349 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.515374+0000", 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.515374+0000", 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515661+0000", 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.350 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830465+0000", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.351 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1a", 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.352 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:03:06.066379+0000", 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.353 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.354 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.355 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.356 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.357 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.399904+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.399904+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.400168+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.358 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859940+0000", 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:25.359 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1b", 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:48:35.387300+0000", 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.360 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.361 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.362 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.363 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.364 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.513309+0000", 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.513309+0000", 2024-09-14T03:37:25.365 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.514049+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831124+0000", 2024-09-14T03:37:25.366 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1b", 2024-09-14T03:37:25.367 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:44:14.127299+0000", 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.368 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.369 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.370 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.371 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.372 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405514+0000", 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405514+0000", 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406006+0000", 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.373 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916208+0000", 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.374 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1a", 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.375 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:23:17.405355+0000", 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.376 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.377 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.378 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.379 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.380 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.398766+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.398766+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399102+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:25.381 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859715+0000", 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.382 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.19", 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:08:01.084059+0000", 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.383 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.384 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.385 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.386 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.387 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519825+0000", 2024-09-14T03:37:25.388 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519825+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520106+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.389 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859784+0000", 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.390 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.18", 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:28:44.762587+0000", 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.391 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.392 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.393 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.394 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.395 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405828+0000", 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405828+0000", 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406587+0000", 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:25.396 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839014+0000", 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.397 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.18", 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.398 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:05:57.857900+0000", 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.399 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.400 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.401 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.402 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.403 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.506311+0000", 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.506311+0000", 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.507004+0000", 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.404 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.836777+0000", 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.405 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.19", 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.406 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:01:33.038366+0000", 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.407 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.408 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.409 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.410 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.411 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400274+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400274+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.402116+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.412 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916191+0000", 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.413 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.17", 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:54:46.213173+0000", 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.414 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.415 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.416 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.417 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.418 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519909+0000", 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519909+0000", 2024-09-14T03:37:25.419 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520187+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839481+0000", 2024-09-14T03:37:25.420 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.16", 2024-09-14T03:37:25.421 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:10:32.588302+0000", 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.422 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.423 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.424 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.425 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.426 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.401698+0000", 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.401698+0000", 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.402069+0000", 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.427 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839588+0000", 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.428 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.16", 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.429 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:16:48.483610+0000", 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.430 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.431 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.432 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.433 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.434 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.515256+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.515256+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515495+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.435 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831335+0000", 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.436 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.17", 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:26:02.392960+0000", 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.437 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.438 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.439 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.440 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.441 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.398281+0000", 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.398281+0000", 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.398566+0000", 2024-09-14T03:37:25.442 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837428+0000", 2024-09-14T03:37:25.443 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 3, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 3, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.15", 2024-09-14T03:37:25.444 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 27, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:00:42.503990+0000", 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 90, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.445 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:25.446 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.447 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 4, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.448 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'3" 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.449 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.953335+0000", 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.953335+0000", 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.953658+0000", 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.450 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830716+0000", 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:25.451 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.14", 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 18, 2024-09-14T03:37:25.452 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:10:40.796694+0000", 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.453 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.454 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.455 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.456 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.457 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400453+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400453+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.401007+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.458 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:36:19.917287+0000", 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 12, 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 12, 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.459 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.14", 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 36, 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:07:32.633267+0000", 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 2624, 2024-09-14T03:37:25.460 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.461 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 12, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 4, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 4, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.462 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 12, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 9 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.463 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "68'12" 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.464 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511328+0000", 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511328+0000", 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.511714+0000", 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.465 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916702+0000", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.466 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.15", 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.467 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:00:06.445335+0000", 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.468 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.469 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.470 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.471 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.472 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.395835+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.395835+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.396984+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:25.473 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831162+0000", 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.474 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.13", 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:49:57.889158+0000", 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.475 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 34, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.476 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.477 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:25.478 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 6 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.479 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.520704+0000", 2024-09-14T03:37:25.480 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.520704+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521192+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:25.481 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830709+0000", 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.482 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.12", 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:37:13.469199+0000", 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.483 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.484 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.485 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.486 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.487 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405882+0000", 2024-09-14T03:37:25.488 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405882+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406737+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:25.489 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839263+0000", 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.490 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.12", 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:00:58.379549+0000", 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.491 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.492 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.493 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.494 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.495 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516007+0000", 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516007+0000", 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517299+0000", 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.496 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916562+0000", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.497 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.13", 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.498 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:16:49.173576+0000", 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.499 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.500 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.501 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.502 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.503 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400073+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400073+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.401544+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:25.504 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916024+0000", 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.505 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.11", 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:48:24.935055+0000", 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.506 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.507 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.508 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.509 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.510 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516657+0000", 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516657+0000", 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516939+0000", 2024-09-14T03:37:25.511 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916207+0000", 2024-09-14T03:37:25.512 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.513 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.10", 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:56:17.423269+0000", 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.514 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.515 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.516 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.517 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.518 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400323+0000", 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400323+0000", 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.400570+0000", 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.519 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830004+0000", 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 8, 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.520 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 8, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.10", 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 32, 2024-09-14T03:37:25.521 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:39:46.484991+0000", 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.522 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 12, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 4, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 4, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.523 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 4, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.524 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 12, 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'8" 2024-09-14T03:37:25.525 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.526 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519973+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519973+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521031+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:25.527 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.915927+0000", 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.528 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.11", 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:12:20.217955+0000", 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.529 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.533 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.534 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.535 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.536 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.537 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.398533+0000", 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.398533+0000", 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399709+0000", 2024-09-14T03:37:25.538 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:25.539 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831096+0000", 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.540 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.f", 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:12:43.525175+0000", 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 22, 2024-09-14T03:37:25.541 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.542 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.543 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 1, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.544 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.545 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.520365+0000", 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.520365+0000", 2024-09-14T03:37:25.546 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520629+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.917039+0000", 2024-09-14T03:37:25.547 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.548 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.e", 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:59:29.519783+0000", 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.549 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.550 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.551 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.552 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.553 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.404261+0000", 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.404261+0000", 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.404858+0000", 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.554 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830800+0000", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.555 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.e", 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.556 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:24:38.834576+0000", 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.557 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.558 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.559 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.560 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.561 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516527+0000", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516527+0000", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516816+0000", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:25.562 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916965+0000", 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.563 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.f", 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:49:35.627973+0000", 2024-09-14T03:37:25.564 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.565 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.566 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.567 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.568 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.569 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.395255+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.395255+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.395532+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.570 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837514+0000", 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:25.571 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.d", 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:36:51.477913+0000", 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 70, 2024-09-14T03:37:25.572 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.573 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.574 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 1, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.575 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.576 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.520420+0000", 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.520420+0000", 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.520744+0000", 2024-09-14T03:37:25.577 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839317+0000", 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.578 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.c", 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.579 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:32:43.573943+0000", 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.580 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.581 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.582 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.583 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.584 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.399008+0000", 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.399008+0000", 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399299+0000", 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.585 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830896+0000", 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.586 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.c", 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:16:14.935721+0000", 2024-09-14T03:37:25.587 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.588 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.589 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.590 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.591 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.592 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.519164+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.519164+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.521443+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.593 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916805+0000", 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.594 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.d", 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:34:28.199628+0000", 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.595 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.596 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.597 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.598 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.599 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.000396+0000", 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.000396+0000", 2024-09-14T03:37:25.600 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.917557+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:25.601 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.293111+0000", 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.602 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.5", 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:18:00.378690+0000", 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000302743, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.603 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.604 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.605 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.606 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.607 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 20, 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:36.967695+0000", 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:36.967695+0000", 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:36.972974+0000", 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:31:39.719769+0000", 2024-09-14T03:37:25.608 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:31:39.719769+0000", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 51, 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:31:39.719769+0000", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:37:20.188212+0000", 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 95, 2024-09-14T03:37:25.609 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 50, 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 95, 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "1.0", 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.610 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 116, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:28:24.258573+0000", 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 590368, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 590368, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.611 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.612 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 2, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.613 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 272, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 479, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 145, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1628 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.614 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "68'95" 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.615 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405473+0000", 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405473+0000", 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406222+0000", 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.616 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839183+0000", 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.617 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.3", 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.618 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:09:26.326984+0000", 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.619 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.620 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.621 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.622 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.623 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512135+0000", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512135+0000", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512416+0000", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:25.624 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859862+0000", 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.625 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.2", 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.626 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:31:49.847882+0000", 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.627 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.628 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.629 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.630 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.631 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.346558+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.346558+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.831437+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:25.632 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:04.980477+0000", 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.633 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.6", 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:15:11.239970+0000", 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000503977, 2024-09-14T03:37:25.634 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.635 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.636 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.637 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.638 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.639 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.408622+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.408622+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.409150+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.640 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916355+0000", 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.641 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.0", 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:11:03.508216+0000", 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.642 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.643 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.644 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.645 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.646 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.510158+0000", 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.510158+0000", 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517068+0000", 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:25.647 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830380+0000", 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.648 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1", 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.649 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:20:22.709122+0000", 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.650 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.651 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.652 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.653 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.654 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.006563+0000", 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.006563+0000", 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.837898+0000", 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:25.655 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.531485+0000", 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.656 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.7", 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:50:37.308715+0000", 2024-09-14T03:37:25.657 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000272495, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.658 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.659 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.660 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.661 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:25.662 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.399698+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.399698+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399987+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.663 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830160+0000", 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.664 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1", 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:26:58.806927+0000", 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.665 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.666 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.667 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.668 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.669 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.515012+0000", 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.515012+0000", 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.515266+0000", 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.670 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.836764+0000", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.671 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.0", 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.672 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:52:23.510282+0000", 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.673 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.674 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.675 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.676 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.677 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.346774+0000", 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.346774+0000", 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.837741+0000", 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:25.678 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.531194+0000", 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:25.679 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.4", 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:25.680 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:36:53.687838+0000", 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000654522, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.681 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.682 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.683 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.684 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.685 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.399434+0000", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.399434+0000", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.399736+0000", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:25.686 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839454+0000", 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.687 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.2", 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.688 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:28:12.790258+0000", 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.689 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.690 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.691 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.692 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.693 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.507452+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.507452+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.507742+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:25.694 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830739+0000", 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.695 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.3", 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:49:50.407482+0000", 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.696 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.697 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.698 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.699 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.700 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.005925+0000", 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.005925+0000", 2024-09-14T03:37:25.701 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.837653+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:25.702 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.530926+0000", 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.703 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.2", 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:57:41.107409+0000", 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000833497, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 19, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.704 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.705 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.706 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 2, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 2 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.707 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "67'2" 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.708 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400204+0000", 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400204+0000", 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.400498+0000", 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.709 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837358+0000", 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.710 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.4", 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.711 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:07:48.735887+0000", 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.712 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.713 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.714 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.715 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.716 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.513689+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.513689+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513933+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:25.717 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839417+0000", 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.718 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.5", 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:37:23.791135+0000", 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.719 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.720 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.721 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.722 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.723 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.005945+0000", 2024-09-14T03:37:25.724 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.005945+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.831590+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:25.725 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:04.979871+0000", 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.726 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.3", 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 20, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:05:00.810273+0000", 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000581876, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.727 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.728 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.729 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.730 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "65'1" 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 2, 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.731 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:34.954209+0000", 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:34.954209+0000", 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:34.954630+0000", 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.732 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 60, 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830345+0000", 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.733 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 59, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.5", 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 18, 2024-09-14T03:37:25.734 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:09:57.878910+0000", 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.735 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.736 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.737 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 2, 2024-09-14T03:37:25.738 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.739 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512679+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512679+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.513000+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:25.740 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837283+0000", 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.741 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.4", 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:43:48.821069+0000", 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.742 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.743 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.744 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.745 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.746 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:51.997457+0000", 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:51.997457+0000", 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.860310+0000", 2024-09-14T03:37:25.747 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.857050+0000", 2024-09-14T03:37:25.748 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.0", 2024-09-14T03:37:25.749 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:38:30.409340+0000", 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000706684, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.750 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.751 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.752 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.753 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.754 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.397377+0000", 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.397377+0000", 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.397629+0000", 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.755 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837791+0000", 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.756 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.6", 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.757 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:42:02.914679+0000", 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.758 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.759 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.760 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 1, 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.761 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.762 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.517367+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.517367+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.517598+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:25.763 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.860019+0000", 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.764 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.7", 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:32:44.727080+0000", 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.765 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.766 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.767 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.768 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.769 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 62, 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:52.006158+0000", 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:52.006158+0000", 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:56.830903+0000", 2024-09-14T03:37:25.770 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 63, 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:50.989704+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:35:05.145236+0000", 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.771 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 62, 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "4.1", 2024-09-14T03:37:25.772 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 68, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 19, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:33:03.846292+0000", 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.000598326, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.773 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.774 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.775 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.776 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.777 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.404994+0000", 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.404994+0000", 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.405301+0000", 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.778 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916770+0000", 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.779 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.7", 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.780 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:52:19.415253+0000", 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.781 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.782 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.783 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.784 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:25.785 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516212+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516212+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516570+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.786 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830839+0000", 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.787 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.6", 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:33:23.282986+0000", 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.788 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.789 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.790 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.791 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.792 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.408521+0000", 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.408521+0000", 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.408805+0000", 2024-09-14T03:37:25.793 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916200+0000", 2024-09-14T03:37:25.794 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.8", 2024-09-14T03:37:25.795 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:20:00.847563+0000", 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.796 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.797 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.798 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.799 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 4, 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.800 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.511131+0000", 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.511131+0000", 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.511411+0000", 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.801 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.830664+0000", 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.802 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.9", 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.803 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:29:23.033860+0000", 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.804 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.805 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.806 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: 2, 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 4, 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.807 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.808 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.400172+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.400172+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.400483+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.809 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.836906+0000", 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.810 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.9", 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:23:40.756478+0000", 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.811 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.812 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.813 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.814 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: 3 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.815 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.505460+0000", 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.505460+0000", 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.505712+0000", 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:25.816 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859865+0000", 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.817 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.8", 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.818 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:08:20.752579+0000", 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.819 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.820 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.821 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.822 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 1, 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.823 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.405382+0000", 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.405382+0000", 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.405640+0000", 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.824 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.837257+0000", 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.825 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.a", 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 24, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.826 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:49:36.350128+0000", 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.827 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.828 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.829 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: 1, 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: 7 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 1, 2024-09-14T03:37:25.830 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.831 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516335+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516335+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516581+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.832 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859846+0000", 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.833 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.b", 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:27:06.070599+0000", 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.834 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.835 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.836 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.837 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.838 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.408264+0000", 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.408264+0000", 2024-09-14T03:37:25.839 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.408573+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916927+0000", 2024-09-14T03:37:25.840 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.841 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.b", 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:40:30.937449+0000", 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.842 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.843 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.844 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 1, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.845 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 6, 2024-09-14T03:37:25.846 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.516119+0000", 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.516119+0000", 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.516408+0000", 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.847 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916037+0000", 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.848 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.a", 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.849 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:38:16.570920+0000", 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.850 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.851 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.852 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 6, 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.853 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.854 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.512183+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.512183+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.512557+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.855 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839391+0000", 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.856 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1d", 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:05:14.073579+0000", 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.857 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.858 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.859 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.860 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: 6 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.861 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.406065+0000", 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.406065+0000", 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406333+0000", 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:25.862 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916723+0000", 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.863 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1c", 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.864 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:22:04.009889+0000", 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.865 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.866 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.867 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.868 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 5, 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.869 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.508086+0000", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.508086+0000", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.508363+0000", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:25.870 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.839294+0000", 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.871 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1c", 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:46:47.194036+0000", 2024-09-14T03:37:25.872 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.873 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.874 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.875 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: 4, 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: 1 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 5, 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.876 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 7, 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:25.877 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.406662+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.406662+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.406945+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.878 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.916839+0000", 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 2, 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 2, 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.879 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1d", 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 26, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:46:42.749128+0000", 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.880 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.881 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 3, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 1, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 1, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.882 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 3, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.883 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: 0 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 7, 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'2" 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.884 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.510663+0000", 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.510663+0000", 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.510922+0000", 2024-09-14T03:37:25.885 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831327+0000", 2024-09-14T03:37:25.886 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1f", 2024-09-14T03:37:25.887 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:00:02.779787+0000", 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.888 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.889 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.890 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.891 INFO:tasks.workunit.client.0.smithi110.stdout: 5, 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.892 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.398725+0000", 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.398725+0000", 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.398992+0000", 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.893 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.860081+0000", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 1, 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.894 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 1, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1e", 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 25, 2024-09-14T03:37:25.895 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:50:20.745864+0000", 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.896 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.897 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.898 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: 5 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'1" 2024-09-14T03:37:25.899 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 3, 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 55, 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.900 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:34:02.514500+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:34:02.514500+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:34:02.514810+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 56, 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.901 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:34:01.486715+0000", 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.859899+0000", 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 55, 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.902 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "3.1e", 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 21, 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:20:21.110046+0000", 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.903 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.904 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.905 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.906 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: 3, 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: 6, 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: 2 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 3, 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "0'0" 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": [ 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.907 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "acting_primary": 0, 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "avail_no_missing": [], 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "blocked_by": [], 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "created": 52, 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "dirty_stats_invalid": false, 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "hitset_stats_invalid": false, 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "last_active": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_active": "2024-09-14T03:33:59.396670+0000", 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "last_became_peered": "2024-09-14T03:33:59.396670+0000", 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "last_change": "2024-09-14T03:33:59.397898+0000", 2024-09-14T03:37:25.908 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_clean_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_deep_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_epoch_clean": 53, 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fresh": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_fullsized": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_peered": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub": "0'0", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_duration": 0, 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_scrub_stamp": "2024-09-14T03:33:58.386582+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_undegraded": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "last_unstale": "2024-09-14T03:34:56.831482+0000", 2024-09-14T03:37:25.909 INFO:tasks.workunit.client.0.smithi110.stdout: "log_dups_size": 0, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 4, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "log_start": "0'0", 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "manifest_stats_invalid": false, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "mapping_epoch": 52, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "object_location_counts": [], 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_scrubbed": 0, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "objects_trimmed": 0, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_stats_invalid": false, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 4, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "parent": "0.0", 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "parent_split_bits": 0, 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "pgid": "2.1f", 2024-09-14T03:37:25.910 INFO:tasks.workunit.client.0.smithi110.stdout: "pin_stats_invalid": false, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "purged_snaps": [], 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_epoch": 67, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "reported_seq": 28, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_duration": 0.0, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:15:46.652243+0000", 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "snaptrimq_len": 0, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 570, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.911 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:25.912 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 1, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.913 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 4, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "state": "active+clean", 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_invalid": false, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: "up": [ 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: 0, 2024-09-14T03:37:25.914 INFO:tasks.workunit.client.0.smithi110.stdout: 7, 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: 4 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: "up_primary": 0, 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: "version": "58'4" 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_per_pool": [ 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 24, 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: "rbd" 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 3, 2024-09-14T03:37:25.915 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pg": 8, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 8, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 3, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "poolid": 4, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 19, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.916 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.917 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.918 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 2, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 2 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 12288, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:25.919 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 57, 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 24 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 192, 2024-09-14T03:37:25.920 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 0, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pg": 64, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 8, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 0, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "poolid": 3, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 0, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.921 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 0, 2024-09-14T03:37:25.922 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.923 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 0 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:25.924 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 0, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 192 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 96, 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:25.925 INFO:tasks.workunit.client.0.smithi110.stdout: "cephfs" 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 45, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pg": 32, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 8, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 45, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "poolid": 2, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 3410, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 0, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.926 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 66, 2024-09-14T03:37:25.927 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 22, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 22, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 14, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.928 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 0, 2024-09-14T03:37:25.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 0, 2024-09-14T03:37:25.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.929 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 51, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 19 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 98304, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 10230, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:25.930 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 96 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: { 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: "acting": 3, 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: "application": [ 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: "mgr" 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: "log_size": 95, 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: "num_pg": 1, 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: "num_store_stats": 3, 2024-09-14T03:37:25.931 INFO:tasks.workunit.client.0.smithi110.stdout: "ondisk_log_size": 95, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "poolid": 1, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "stat_sum": { 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes": 590368, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_bytes_recovered": 590368, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict": 0, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_kb": 0, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_full": 0, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_evict_mode_some": 0, 2024-09-14T03:37:25.932 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_kb": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_high": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_flush_mode_low": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_keys_recovered": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_large_omap_objects": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_clones": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_object_copies": 6, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects": 2, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_degraded": 0, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_dirty": 2, 2024-09-14T03:37:25.933 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_manifest": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_misplaced": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_omap": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_pinned": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_recovered": 2, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_repaired": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_objects_unfound": 0, 2024-09-14T03:37:25.934 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_bytes": 0, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_omap_keys": 0, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_promote": 0, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read": 272, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_read_kb": 479, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_scrub_errors": 0, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_whiteouts": 0, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write": 145, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "num_write_kb": 1628 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.935 INFO:tasks.workunit.client.0.smithi110.stdout: "store_stats": { 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "allocated": 1781760, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "available": 0, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed": 0, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_allocated": 0, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "data_compressed_original": 0, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "data_stored": 1771104, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "internal_metadata": 0, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "internally_reserved": 0, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "omap_allocated": 0, 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: "total": 0 2024-09-14T03:37:25.936 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.937 INFO:tasks.workunit.client.0.smithi110.stdout: "up": 3 2024-09-14T03:37:25.937 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.937 INFO:tasks.workunit.client.0.smithi110.stdout: ], 2024-09-14T03:37:25.937 INFO:tasks.workunit.client.0.smithi110.stdout: "usage": { 2024-09-14T03:37:25.937 INFO:tasks.workunit.client.0.smithi110.stdout: "pg_num": 105, 2024-09-14T03:37:25.937 INFO:tasks.workunit.client.0.smithi110.stdout: "pools": 4, 2024-09-14T03:37:25.940 INFO:tasks.workunit.client.0.smithi110.stdout: "stats_by_class": { 2024-09-14T03:37:25.940 INFO:tasks.workunit.client.0.smithi110.stdout: "ssd": { 2024-09-14T03:37:25.940 INFO:tasks.workunit.client.0.smithi110.stdout: "total_avail_bytes": 765516926976, 2024-09-14T03:37:25.940 INFO:tasks.workunit.client.0.smithi110.stdout: "total_bytes": 767960285184, 2024-09-14T03:37:25.940 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_bytes": 2443358208, 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_raw_bytes": 2443358208, 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_raw_ratio": 0.0031816205009818077 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: }, 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: "total_avail_bytes": 765516926976, 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: "total_bytes": 767960285184, 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: "total_used_bytes": 2443358208 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.941 INFO:tasks.workunit.client.0.smithi110.stdout: } 2024-09-14T03:37:25.942 INFO:tasks.workunit.client.0.smithi110.stdout:} 2024-09-14T03:37:25.942 INFO:tasks.workunit.client.0.smithi110.stdout:OK 2024-09-14T03:37:25.942 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:37:25.942 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:37:25.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.945 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.945 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.945 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.945 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.945 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.945 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.945 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:24 smithi195 ceph-mon[30695]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:24 smithi195 ceph-mon[30695]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:25.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:37:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:37:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:23 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:24 smithi110 ceph-mon[35374]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:24 smithi110 ceph-mon[35374]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:24 smithi110 ceph-mon[27365]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:24 smithi110 ceph-mon[27365]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:25.988 INFO:tasks.workunit:Stopping ['test_telemetry_quincy.sh'] on client.0... 2024-09-14T03:37:25.988 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-14T03:37:26.297 DEBUG:teuthology.parallel:result is None 2024-09-14T03:37:26.298 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T03:37:26.323 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T03:37:26.323 DEBUG:teuthology.orchestra.run.smithi110:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:37:26.383 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T03:37:26.384 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:37:26.397 INFO:teuthology.task.print:**** done end telemetry quincy... 2024-09-14T03:37:26.397 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:37:26.440 INFO:teuthology.task.print:**** done start parallel 2024-09-14T03:37:26.440 INFO:teuthology.run_tasks:Running task parallel... 2024-09-14T03:37:26.476 INFO:teuthology.task.parallel:starting parallel... 2024-09-14T03:37:26.476 INFO:teuthology.task.parallel:In parallel, running task full_sequential... 2024-09-14T03:37:26.478 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-14T03:37:26.478 INFO:teuthology.task.print:**** done start ec-rados-default.yaml 2024-09-14T03:37:26.478 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-09-14T03:37:26.481 INFO:tasks.rados:Beginning rados... 2024-09-14T03:37:26.481 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-14T03:37:26.482 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '1c941a66c1b420259b8fc5ac19e792dbbf60aded'}} 2024-09-14T03:37:26.482 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-14T03:37:26.482 INFO:tasks.rados:joining rados 2024-09-14T03:37:26.482 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-14T03:37:26.483 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-14T03:37:26.483 INFO:teuthology.task.print:**** done start upgrade, wait 2024-09-14T03:37:26.483 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-14T03:37:26.483 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi110.front.sepia.ceph.com 2024-09-14T03:37:26.484 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-14T03:37:26.485 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-09-14T03:37:26.485 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd erasure-code-profile set teuthologyprofile k=2 m=1 crush-failure-domain=osd 2024-09-14T03:37:26.746 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:26.748 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:27.127 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:26 smithi110 ceph-mon[35374]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:27.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:26 smithi110 ceph-mon[27365]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:27.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:26 smithi195 ceph-mon[30695]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:37:29.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:28 smithi110 ceph-mon[35374]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:37:29.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:28 smithi110 ceph-mon[27365]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:37:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:28 smithi195 ceph-mon[30695]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:37:30.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-14T03:37:30.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:37:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:37:30.514 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:30.637 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:30] "GET /metrics HTTP/1.1" 200 236472 "" "Prometheus/2.43.0" 2024-09-14T03:37:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:30 smithi110 ceph-mon[27365]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:37:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:30 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/317709255' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:37:31.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:30 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:37:31.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:30 smithi110 ceph-mon[35374]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:37:31.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:30 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/317709255' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:37:31.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:30 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:37:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:30 smithi195 ceph-mon[30695]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:37:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:30 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/317709255' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:37:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:30 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:37:31.701 INFO:tasks.rados:starting run 0 out of 1 2024-09-14T03:37:31.701 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-14T03:37:31.701 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 16 erasure teuthologyprofile 2024-09-14T03:37:32.254 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:31 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-14T03:37:32.255 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:31 smithi110 ceph-mon[35374]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:37:32.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:31 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-14T03:37:32.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:31 smithi110 ceph-mon[27365]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:37:32.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:31 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-14T03:37:32.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:31 smithi195 ceph-mon[30695]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:37:32.608 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:32.772 INFO:teuthology.orchestra.run.smithi110.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[35374]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:37:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[27365]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:37:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:33.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:32 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:33.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:37:33.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:32 smithi195 ceph-mon[30695]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:37:33.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:32 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:32 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:32 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:32 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:32 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:33 smithi110 ceph-mon[35374]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:33 smithi110 ceph-mon[35374]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:33 smithi110 ceph-mon[35374]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:33 smithi110 ceph-mon[27365]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:33 smithi110 ceph-mon[27365]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:33 smithi110 ceph-mon[27365]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:33 smithi195 ceph-mon[30695]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:33 smithi195 ceph-mon[30695]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:33 smithi195 ceph-mon[30695]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:34.859 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:35.245 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:34 smithi110 ceph-mon[35374]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:37:35.245 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:34 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:35.245 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:34 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/1937109928' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:37:35.246 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:34 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:37:35.246 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:34 smithi110 ceph-mon[27365]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:37:35.246 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:34 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:35.246 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:34 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/1937109928' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:37:35.246 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:34 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:37:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:34 smithi195 ceph-mon[30695]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:37:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:34 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:34 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/1937109928' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:37:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:34 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:37:36.012 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_0' created 2024-09-14T03:37:36.383 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:36 smithi110 ceph-mon[35374]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:37:36.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:36 smithi110 ceph-mon[27365]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:37:36.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:36 smithi195 ceph-mon[30695]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:37:36.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-14T03:37:37.192 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:37 smithi110 ceph-mon[35374]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:37 smithi110 ceph-mon[35374]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-14T03:37:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:37 smithi110 ceph-mon[35374]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:37:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:37 smithi110 ceph-mon[27365]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:37 smithi110 ceph-mon[27365]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-14T03:37:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:37 smithi110 ceph-mon[27365]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:37:37.391 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:37:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:37 smithi195 ceph-mon[30695]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:37 smithi195 ceph-mon[30695]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-14T03:37:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:37 smithi195 ceph-mon[30695]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:37:37.985 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:37:37.985 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (98s) 78s ago 2m 18.3M - 0.25.0 c8568f914cd2 764997e49f35 2024-09-14T03:37:37.985 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (2m) 83s ago 2m 80.5M - 9.4.7 954c08fa6188 67ea5066f839 2024-09-14T03:37:37.985 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (3m) 78s ago 3m 16.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:37:37.985 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (3m) 83s ago 3m 15.0M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:37:37.985 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443 running (7m) 83s ago 7m 399M - 17.2.7-1629-geb16b97b ab51d1bebd12 68be7f1bb797 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:9283,8765 running (8m) 78s ago 8m 446M - 17.2.7-1629-geb16b97b ab51d1bebd12 a83ea41d9e62 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (8m) 78s ago 8m 44.8M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 c1073676c66c 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (7m) 83s ago 7m 40.7M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 b6415a0de807 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (7m) 78s ago 7m 39.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 8e2894672cd8 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (2m) 78s ago 2m 13.2M - 1.5.0 0da6a335fe13 986afca49027 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (2m) 83s ago 2m 3359k - 1.5.0 0da6a335fe13 67a66167cf52 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (6m) 78s ago 6m 43.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (6m) 78s ago 6m 43.8M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (6m) 78s ago 6m 43.1M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (5m) 78s ago 5m 41.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (5m) 83s ago 5m 41.7M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (4m) 83s ago 4m 42.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (4m) 83s ago 4m 41.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:37:37.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (4m) 83s ago 4m 42.8M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:37:37.987 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (92s) 83s ago 2m 24.8M - 2.43.0 a07b618ecd1d 4460d604b8f6 2024-09-14T03:37:38.342 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:37:38.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:37:38.344 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:37:38.344 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 15 2024-09-14T03:37:38.344 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:37:38.344 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:37:38.344 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:38 smithi110 ceph-mon[27365]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:37:38.344 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:38 smithi110 ceph-mon[27365]: from='client.24800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:38.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:38 smithi110 ceph-mon[35374]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:37:38.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:38 smithi110 ceph-mon[35374]: from='client.24800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:38.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:38 smithi195 ceph-mon[30695]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:37:38.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:38 smithi195 ceph-mon[30695]: from='client.24800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:38.645 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:37:38.645 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", 2024-09-14T03:37:38.645 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:37:38.646 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:37:38.646 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [], 2024-09-14T03:37:38.646 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "", 2024-09-14T03:37:38.646 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 image", 2024-09-14T03:37:38.646 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:37:38.646 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:37:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[27365]: from='client.14916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[27365]: pgmap v98: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[27365]: from='client.14922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[27365]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:37:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[27365]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:37:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/568839956' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:37:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[35374]: from='client.14916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[35374]: pgmap v98: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[35374]: from='client.14922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[35374]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:37:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[35374]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:37:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:39 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/568839956' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:37:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:39 smithi195 ceph-mon[30695]: from='client.14916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:39 smithi195 ceph-mon[30695]: pgmap v98: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:39 smithi195 ceph-mon[30695]: from='client.14922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:39 smithi195 ceph-mon[30695]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:37:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:39 smithi195 ceph-mon[30695]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:37:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:39 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/568839956' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:37:40.025 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-14T03:37:40.343 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:40 smithi110 ceph-mon[35374]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:40.343 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:40 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/922638884' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:37:40.344 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:40 smithi110 ceph-mon[27365]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:40.344 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:40 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/922638884' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:37:40.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:40 smithi195 ceph-mon[30695]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:40.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:40 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/922638884' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:37:40.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:37:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:37:40.664 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:40 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:40] "GET /metrics HTTP/1.1" 200 236505 "" "Prometheus/2.43.0" 2024-09-14T03:37:41.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:41 smithi110 ceph-mon[27365]: pgmap v100: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:41.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:41 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/922638884' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:37:41.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:41 smithi110 ceph-mon[27365]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:37:41.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:41 smithi110 ceph-mon[35374]: pgmap v100: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:41.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:41 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/922638884' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:37:41.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:41 smithi110 ceph-mon[35374]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:37:41.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:41 smithi195 ceph-mon[30695]: pgmap v100: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:41 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/922638884' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:37:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:41 smithi195 ceph-mon[30695]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:37:41.691 DEBUG:teuthology.orchestra.run.smithi110:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op copy_from 50 --op append 100 --pool unique_pool_0 2024-09-14T03:37:41.717 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight read -> 100 2024-09-14T03:37:41.717 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight delete -> 50 2024-09-14T03:37:41.717 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight snap_create -> 50 2024-09-14T03:37:41.717 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight snap_remove -> 50 2024-09-14T03:37:41.717 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight rollback -> 50 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight setattr -> 25 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight rmattr -> 25 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight copy_from -> 50 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout:adding op weight append -> 100 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout:ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable) 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout:Configuration: 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout: Number of operations: 4000 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout: Number of objects: 50 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout: Max in flight operations: 16 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout: Object size (in bytes): 4000000 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout: Write stride min: 400000 2024-09-14T03:37:41.718 INFO:tasks.rados.rados.0.smithi110.stdout: Write stride max: 800000 2024-09-14T03:37:42.030 INFO:tasks.rados.rados.0.smithi110.stdout:1: write initial oid 1 2024-09-14T03:37:42.031 INFO:tasks.rados.rados.0.smithi110.stdout:1: seq_num 0 ranges {0=24576} 2024-09-14T03:37:42.031 INFO:tasks.rados.rados.0.smithi110.stdout:1: writing smithi11075562-1 from 0 to 24576 tid 1 2024-09-14T03:37:42.031 INFO:tasks.rados.rados.0.smithi110.stdout:2: write initial oid 2 2024-09-14T03:37:42.031 INFO:tasks.rados.rados.0.smithi110.stdout:2: seq_num 1 ranges {0=24576} 2024-09-14T03:37:42.031 INFO:tasks.rados.rados.0.smithi110.stdout:2: writing smithi11075562-2 from 0 to 24576 tid 1 2024-09-14T03:37:42.031 INFO:tasks.rados.rados.0.smithi110.stdout:3: write initial oid 3 2024-09-14T03:37:42.031 INFO:tasks.rados.rados.0.smithi110.stdout:3: seq_num 2 ranges {0=40960} 2024-09-14T03:37:42.031 INFO:tasks.rados.rados.0.smithi110.stdout:3: writing smithi11075562-3 from 0 to 40960 tid 1 2024-09-14T03:37:42.032 INFO:tasks.rados.rados.0.smithi110.stdout:4: write initial oid 4 2024-09-14T03:37:42.032 INFO:tasks.rados.rados.0.smithi110.stdout:4: seq_num 3 ranges {0=57344} 2024-09-14T03:37:42.032 INFO:tasks.rados.rados.0.smithi110.stdout:4: writing smithi11075562-4 from 0 to 57344 tid 1 2024-09-14T03:37:42.032 INFO:tasks.rados.rados.0.smithi110.stdout:5: write initial oid 5 2024-09-14T03:37:42.032 INFO:tasks.rados.rados.0.smithi110.stdout:5: seq_num 4 ranges {0=73728} 2024-09-14T03:37:42.033 INFO:tasks.rados.rados.0.smithi110.stdout:5: writing smithi11075562-5 from 0 to 73728 tid 1 2024-09-14T03:37:42.033 INFO:tasks.rados.rados.0.smithi110.stdout:6: write initial oid 6 2024-09-14T03:37:42.034 INFO:tasks.rados.rados.0.smithi110.stdout:6: seq_num 5 ranges {0=90112} 2024-09-14T03:37:42.034 INFO:tasks.rados.rados.0.smithi110.stdout:6: writing smithi11075562-6 from 0 to 90112 tid 1 2024-09-14T03:37:42.034 INFO:tasks.rados.rados.0.smithi110.stdout:7: write initial oid 7 2024-09-14T03:37:42.034 INFO:tasks.rados.rados.0.smithi110.stdout:7: seq_num 6 ranges {0=106496} 2024-09-14T03:37:42.034 INFO:tasks.rados.rados.0.smithi110.stdout:7: writing smithi11075562-7 from 0 to 106496 tid 1 2024-09-14T03:37:42.034 INFO:tasks.rados.rados.0.smithi110.stdout:8: write initial oid 8 2024-09-14T03:37:42.034 INFO:tasks.rados.rados.0.smithi110.stdout:8: seq_num 7 ranges {0=122880} 2024-09-14T03:37:42.035 INFO:tasks.rados.rados.0.smithi110.stdout:8: writing smithi11075562-8 from 0 to 122880 tid 1 2024-09-14T03:37:42.035 INFO:tasks.rados.rados.0.smithi110.stdout:9: write initial oid 9 2024-09-14T03:37:42.035 INFO:tasks.rados.rados.0.smithi110.stdout:9: seq_num 8 ranges {0=139264} 2024-09-14T03:37:42.035 INFO:tasks.rados.rados.0.smithi110.stdout:9: writing smithi11075562-9 from 0 to 139264 tid 1 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:2: finishing write tid 1 to smithi11075562-2 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:1: finishing write tid 1 to smithi11075562-1 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:1: finishing write tid 2 to smithi11075562-1 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:2: finishing write tid 2 to smithi11075562-2 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:1: finishing write tid 3 to smithi11075562-1 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:1: done (8 left) 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:10: write initial oid 10 2024-09-14T03:37:42.036 INFO:tasks.rados.rados.0.smithi110.stdout:10: seq_num 9 ranges {0=155648} 2024-09-14T03:37:42.037 INFO:tasks.rados.rados.0.smithi110.stdout:10: writing smithi11075562-10 from 0 to 155648 tid 1 2024-09-14T03:37:42.037 INFO:tasks.rados.rados.0.smithi110.stdout:11: write initial oid 11 2024-09-14T03:37:42.037 INFO:tasks.rados.rados.0.smithi110.stdout:11: seq_num 10 ranges {0=172032} 2024-09-14T03:37:42.038 INFO:tasks.rados.rados.0.smithi110.stdout:11: writing smithi11075562-11 from 0 to 172032 tid 1 2024-09-14T03:37:42.038 INFO:tasks.rados.rados.0.smithi110.stdout:12: write initial oid 12 2024-09-14T03:37:42.038 INFO:tasks.rados.rados.0.smithi110.stdout:12: seq_num 11 ranges {0=188416} 2024-09-14T03:37:42.040 INFO:tasks.rados.rados.0.smithi110.stdout:12: writing smithi11075562-12 from 0 to 188416 tid 1 2024-09-14T03:37:42.040 INFO:tasks.rados.rados.0.smithi110.stdout:13: write initial oid 13 2024-09-14T03:37:42.040 INFO:tasks.rados.rados.0.smithi110.stdout:13: seq_num 12 ranges {0=204800} 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:13: writing smithi11075562-13 from 0 to 204800 tid 1 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:14: write initial oid 14 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:3: finishing write tid 1 to smithi11075562-3 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:3: finishing write tid 2 to smithi11075562-3 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:6: finishing write tid 1 to smithi11075562-6 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:6: finishing write tid 2 to smithi11075562-6 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:2: finishing write tid 3 to smithi11075562-2 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-14T03:37:42.042 INFO:tasks.rados.rados.0.smithi110.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:5: finishing write tid 1 to smithi11075562-5 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:6: finishing write tid 3 to smithi11075562-6 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:3: finishing write tid 3 to smithi11075562-3 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 4 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:4: finishing write tid 1 to smithi11075562-4 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:8: finishing write tid 1 to smithi11075562-8 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:8: finishing write tid 2 to smithi11075562-8 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:5: finishing write tid 2 to smithi11075562-5 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:5: finishing write tid 3 to smithi11075562-5 2024-09-14T03:37:42.043 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:7: finishing write tid 1 to smithi11075562-7 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:7: finishing write tid 2 to smithi11075562-7 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:4: finishing write tid 2 to smithi11075562-4 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:8: finishing write tid 3 to smithi11075562-8 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 2 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:9: finishing write tid 1 to smithi11075562-9 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:10: finishing write tid 1 to smithi11075562-10 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:7: finishing write tid 3 to smithi11075562-7 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:37:42.044 INFO:tasks.rados.rados.0.smithi110.stdout:14: seq_num 13 ranges {0=221184} 2024-09-14T03:37:42.045 INFO:tasks.rados.rados.0.smithi110.stdout:14: writing smithi11075562-14 from 0 to 221184 tid 1 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:2: done (12 left) 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:3: done (11 left) 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:5: done (10 left) 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:6: done (9 left) 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:7: done (8 left) 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:8: done (7 left) 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:15: write initial oid 15 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:15: seq_num 14 ranges {0=237568} 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:15: writing smithi11075562-15 from 0 to 237568 tid 1 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:16: write initial oid 16 2024-09-14T03:37:42.046 INFO:tasks.rados.rados.0.smithi110.stdout:16: seq_num 15 ranges {0=253952} 2024-09-14T03:37:42.047 INFO:tasks.rados.rados.0.smithi110.stdout:16: writing smithi11075562-16 from 0 to 253952 tid 1 2024-09-14T03:37:42.047 INFO:tasks.rados.rados.0.smithi110.stdout:17: write initial oid 17 2024-09-14T03:37:42.047 INFO:tasks.rados.rados.0.smithi110.stdout:10: finishing write tid 2 to smithi11075562-10 2024-09-14T03:37:42.047 INFO:tasks.rados.rados.0.smithi110.stdout:4: finishing write tid 3 to smithi11075562-4 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:10: finishing write tid 3 to smithi11075562-10 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:11: finishing write tid 1 to smithi11075562-11 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:11: finishing write tid 2 to smithi11075562-11 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:12: finishing write tid 1 to smithi11075562-12 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:9: finishing write tid 2 to smithi11075562-9 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:12: finishing write tid 2 to smithi11075562-12 2024-09-14T03:37:42.048 INFO:tasks.rados.rados.0.smithi110.stdout:11: finishing write tid 3 to smithi11075562-11 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 2 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:13: finishing write tid 1 to smithi11075562-13 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:9: finishing write tid 3 to smithi11075562-9 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 2 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:13: finishing write tid 2 to smithi11075562-13 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:12: finishing write tid 3 to smithi11075562-12 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 4 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-14T03:37:42.049 INFO:tasks.rados.rados.0.smithi110.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-09-14T03:37:42.050 INFO:tasks.rados.rados.0.smithi110.stdout:14: finishing write tid 1 to smithi11075562-14 2024-09-14T03:37:42.050 INFO:tasks.rados.rados.0.smithi110.stdout:13: finishing write tid 3 to smithi11075562-13 2024-09-14T03:37:42.050 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 4 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-14T03:37:42.050 INFO:tasks.rados.rados.0.smithi110.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-09-14T03:37:42.050 INFO:tasks.rados.rados.0.smithi110.stdout:14: finishing write tid 2 to smithi11075562-14 2024-09-14T03:37:42.050 INFO:tasks.rados.rados.0.smithi110.stdout:17: seq_num 16 ranges {0=270336} 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:17: writing smithi11075562-17 from 0 to 270336 tid 1 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:4: done (9 left) 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:9: done (8 left) 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:10: done (7 left) 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:11: done (6 left) 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:12: done (5 left) 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:13: done (4 left) 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:18: write initial oid 18 2024-09-14T03:37:42.051 INFO:tasks.rados.rados.0.smithi110.stdout:18: seq_num 17 ranges {0=286720} 2024-09-14T03:37:42.052 INFO:tasks.rados.rados.0.smithi110.stdout:18: writing smithi11075562-18 from 0 to 286720 tid 1 2024-09-14T03:37:42.052 INFO:tasks.rados.rados.0.smithi110.stdout:19: write initial oid 19 2024-09-14T03:37:42.052 INFO:tasks.rados.rados.0.smithi110.stdout:19: seq_num 18 ranges {0=303104} 2024-09-14T03:37:42.052 INFO:tasks.rados.rados.0.smithi110.stdout:19: writing smithi11075562-19 from 0 to 303104 tid 1 2024-09-14T03:37:42.052 INFO:tasks.rados.rados.0.smithi110.stdout:20: write initial oid 20 2024-09-14T03:37:42.052 INFO:tasks.rados.rados.0.smithi110.stdout:20: seq_num 19 ranges {0=319488} 2024-09-14T03:37:42.054 INFO:tasks.rados.rados.0.smithi110.stdout:20: writing smithi11075562-20 from 0 to 319488 tid 1 2024-09-14T03:37:42.054 INFO:tasks.rados.rados.0.smithi110.stdout:21: write initial oid 21 2024-09-14T03:37:42.054 INFO:tasks.rados.rados.0.smithi110.stdout:21: seq_num 20 ranges {0=344064} 2024-09-14T03:37:42.056 INFO:tasks.rados.rados.0.smithi110.stdout:21: writing smithi11075562-21 from 0 to 344064 tid 1 2024-09-14T03:37:42.056 INFO:tasks.rados.rados.0.smithi110.stdout:22: write initial oid 22 2024-09-14T03:37:42.056 INFO:tasks.rados.rados.0.smithi110.stdout:22: seq_num 21 ranges {0=360448} 2024-09-14T03:37:42.059 INFO:tasks.rados.rados.0.smithi110.stdout:22: writing smithi11075562-22 from 0 to 360448 tid 1 2024-09-14T03:37:42.059 INFO:tasks.rados.rados.0.smithi110.stdout:23: write initial oid 23 2024-09-14T03:37:42.059 INFO:tasks.rados.rados.0.smithi110.stdout:23: seq_num 22 ranges {0=376832} 2024-09-14T03:37:42.061 INFO:tasks.rados.rados.0.smithi110.stdout:23: writing smithi11075562-23 from 0 to 376832 tid 1 2024-09-14T03:37:42.061 INFO:tasks.rados.rados.0.smithi110.stdout:24: write initial oid 24 2024-09-14T03:37:42.061 INFO:tasks.rados.rados.0.smithi110.stdout:24: seq_num 23 ranges {0=393216} 2024-09-14T03:37:42.064 INFO:tasks.rados.rados.0.smithi110.stdout:24: writing smithi11075562-24 from 0 to 393216 tid 1 2024-09-14T03:37:42.064 INFO:tasks.rados.rados.0.smithi110.stdout:25: write initial oid 25 2024-09-14T03:37:42.064 INFO:tasks.rados.rados.0.smithi110.stdout:25: seq_num 24 ranges {0=409600} 2024-09-14T03:37:42.067 INFO:tasks.rados.rados.0.smithi110.stdout:25: writing smithi11075562-25 from 0 to 409600 tid 1 2024-09-14T03:37:42.067 INFO:tasks.rados.rados.0.smithi110.stdout:26: write initial oid 26 2024-09-14T03:37:42.067 INFO:tasks.rados.rados.0.smithi110.stdout:26: seq_num 25 ranges {0=425984} 2024-09-14T03:37:42.069 INFO:tasks.rados.rados.0.smithi110.stdout:26: writing smithi11075562-26 from 0 to 425984 tid 1 2024-09-14T03:37:42.070 INFO:tasks.rados.rados.0.smithi110.stdout:27: write initial oid 27 2024-09-14T03:37:42.070 INFO:tasks.rados.rados.0.smithi110.stdout:27: seq_num 26 ranges {0=442368} 2024-09-14T03:37:42.072 INFO:tasks.rados.rados.0.smithi110.stdout:27: writing smithi11075562-27 from 0 to 442368 tid 1 2024-09-14T03:37:42.072 INFO:tasks.rados.rados.0.smithi110.stdout:28: write initial oid 28 2024-09-14T03:37:42.073 INFO:tasks.rados.rados.0.smithi110.stdout:28: seq_num 27 ranges {0=458752} 2024-09-14T03:37:42.075 INFO:tasks.rados.rados.0.smithi110.stdout:28: writing smithi11075562-28 from 0 to 458752 tid 1 2024-09-14T03:37:42.075 INFO:tasks.rados.rados.0.smithi110.stdout:29: write initial oid 29 2024-09-14T03:37:42.075 INFO:tasks.rados.rados.0.smithi110.stdout:29: seq_num 28 ranges {0=475136} 2024-09-14T03:37:42.079 INFO:tasks.rados.rados.0.smithi110.stdout:29: writing smithi11075562-29 from 0 to 475136 tid 1 2024-09-14T03:37:42.079 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.079 INFO:tasks.rados.rados.0.smithi110.stdout:15: finishing write tid 1 to smithi11075562-15 2024-09-14T03:37:42.079 INFO:tasks.rados.rados.0.smithi110.stdout:15: finishing write tid 2 to smithi11075562-15 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:14: finishing write tid 3 to smithi11075562-14 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 2 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:15: finishing write tid 3 to smithi11075562-15 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 4 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:16: finishing write tid 1 to smithi11075562-16 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:16: finishing write tid 2 to smithi11075562-16 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:16: finishing write tid 3 to smithi11075562-16 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 4 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:17: finishing write tid 1 to smithi11075562-17 2024-09-14T03:37:42.080 INFO:tasks.rados.rados.0.smithi110.stdout:17: finishing write tid 2 to smithi11075562-17 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:17: finishing write tid 3 to smithi11075562-17 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 4 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:18: finishing write tid 1 to smithi11075562-18 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:18: finishing write tid 2 to smithi11075562-18 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:19: finishing write tid 1 to smithi11075562-19 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:14: done (15 left) 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:15: done (14 left) 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:16: done (13 left) 2024-09-14T03:37:42.081 INFO:tasks.rados.rados.0.smithi110.stdout:17: done (12 left) 2024-09-14T03:37:42.082 INFO:tasks.rados.rados.0.smithi110.stdout:30: write initial oid 30 2024-09-14T03:37:42.082 INFO:tasks.rados.rados.0.smithi110.stdout:30: seq_num 29 ranges {0=491520} 2024-09-14T03:37:42.082 INFO:tasks.rados.rados.0.smithi110.stdout:30: writing smithi11075562-30 from 0 to 491520 tid 1 2024-09-14T03:37:42.082 INFO:tasks.rados.rados.0.smithi110.stdout:31: write initial oid 31 2024-09-14T03:37:42.083 INFO:tasks.rados.rados.0.smithi110.stdout:31: seq_num 30 ranges {0=507904} 2024-09-14T03:37:42.085 INFO:tasks.rados.rados.0.smithi110.stdout:31: writing smithi11075562-31 from 0 to 507904 tid 1 2024-09-14T03:37:42.085 INFO:tasks.rados.rados.0.smithi110.stdout:32: write initial oid 32 2024-09-14T03:37:42.086 INFO:tasks.rados.rados.0.smithi110.stdout:32: seq_num 31 ranges {0=524288} 2024-09-14T03:37:42.089 INFO:tasks.rados.rados.0.smithi110.stdout:32: writing smithi11075562-32 from 0 to 524288 tid 1 2024-09-14T03:37:42.089 INFO:tasks.rados.rados.0.smithi110.stdout:33: write initial oid 33 2024-09-14T03:37:42.089 INFO:tasks.rados.rados.0.smithi110.stdout:33: seq_num 32 ranges {0=540672} 2024-09-14T03:37:42.093 INFO:tasks.rados.rados.0.smithi110.stdout:33: writing smithi11075562-33 from 0 to 540672 tid 1 2024-09-14T03:37:42.093 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.093 INFO:tasks.rados.rados.0.smithi110.stdout:19: finishing write tid 2 to smithi11075562-19 2024-09-14T03:37:42.093 INFO:tasks.rados.rados.0.smithi110.stdout:18: finishing write tid 3 to smithi11075562-18 2024-09-14T03:37:42.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 6 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-14T03:37:42.093 INFO:tasks.rados.rados.0.smithi110.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:19: finishing write tid 3 to smithi11075562-19 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 8 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:20: finishing write tid 1 to smithi11075562-20 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:20: finishing write tid 2 to smithi11075562-20 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:18: done (15 left) 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:19: done (14 left) 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:34: write initial oid 34 2024-09-14T03:37:42.094 INFO:tasks.rados.rados.0.smithi110.stdout:34: seq_num 33 ranges {0=557056} 2024-09-14T03:37:42.096 INFO:tasks.rados.rados.0.smithi110.stdout:34: writing smithi11075562-34 from 0 to 557056 tid 1 2024-09-14T03:37:42.096 INFO:tasks.rados.rados.0.smithi110.stdout:35: write initial oid 35 2024-09-14T03:37:42.096 INFO:tasks.rados.rados.0.smithi110.stdout:35: seq_num 34 ranges {0=573440} 2024-09-14T03:37:42.099 INFO:tasks.rados.rados.0.smithi110.stdout:35: writing smithi11075562-35 from 0 to 573440 tid 1 2024-09-14T03:37:42.099 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.100 INFO:tasks.rados.rados.0.smithi110.stdout:20: finishing write tid 3 to smithi11075562-20 2024-09-14T03:37:42.100 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 4 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-14T03:37:42.100 INFO:tasks.rados.rados.0.smithi110.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-09-14T03:37:42.100 INFO:tasks.rados.rados.0.smithi110.stdout:20: done (15 left) 2024-09-14T03:37:42.100 INFO:tasks.rados.rados.0.smithi110.stdout:36: write initial oid 36 2024-09-14T03:37:42.100 INFO:tasks.rados.rados.0.smithi110.stdout:36: seq_num 35 ranges {0=589824} 2024-09-14T03:37:42.103 INFO:tasks.rados.rados.0.smithi110.stdout:36: writing smithi11075562-36 from 0 to 589824 tid 1 2024-09-14T03:37:42.103 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.103 INFO:tasks.rados.rados.0.smithi110.stdout:21: finishing write tid 1 to smithi11075562-21 2024-09-14T03:37:42.103 INFO:tasks.rados.rados.0.smithi110.stdout:21: finishing write tid 2 to smithi11075562-21 2024-09-14T03:37:42.103 INFO:tasks.rados.rados.0.smithi110.stdout:21: finishing write tid 3 to smithi11075562-21 2024-09-14T03:37:42.103 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 4 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-14T03:37:42.103 INFO:tasks.rados.rados.0.smithi110.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-09-14T03:37:42.103 INFO:tasks.rados.rados.0.smithi110.stdout:22: finishing write tid 1 to smithi11075562-22 2024-09-14T03:37:42.104 INFO:tasks.rados.rados.0.smithi110.stdout:22: finishing write tid 2 to smithi11075562-22 2024-09-14T03:37:42.104 INFO:tasks.rados.rados.0.smithi110.stdout:21: done (15 left) 2024-09-14T03:37:42.104 INFO:tasks.rados.rados.0.smithi110.stdout:37: write initial oid 37 2024-09-14T03:37:42.104 INFO:tasks.rados.rados.0.smithi110.stdout:37: seq_num 36 ranges {0=606208} 2024-09-14T03:37:42.106 INFO:tasks.rados.rados.0.smithi110.stdout:37: writing smithi11075562-37 from 0 to 606208 tid 1 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:22: finishing write tid 3 to smithi11075562-22 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 6 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:23: finishing write tid 1 to smithi11075562-23 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:23: finishing write tid 2 to smithi11075562-23 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:23: finishing write tid 3 to smithi11075562-23 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 4 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-09-14T03:37:42.107 INFO:tasks.rados.rados.0.smithi110.stdout:24: finishing write tid 1 to smithi11075562-24 2024-09-14T03:37:42.108 INFO:tasks.rados.rados.0.smithi110.stdout:22: done (15 left) 2024-09-14T03:37:42.108 INFO:tasks.rados.rados.0.smithi110.stdout:23: done (14 left) 2024-09-14T03:37:42.108 INFO:tasks.rados.rados.0.smithi110.stdout:38: write initial oid 38 2024-09-14T03:37:42.108 INFO:tasks.rados.rados.0.smithi110.stdout:38: seq_num 37 ranges {0=622592} 2024-09-14T03:37:42.110 INFO:tasks.rados.rados.0.smithi110.stdout:38: writing smithi11075562-38 from 0 to 622592 tid 1 2024-09-14T03:37:42.110 INFO:tasks.rados.rados.0.smithi110.stdout:39: write initial oid 39 2024-09-14T03:37:42.111 INFO:tasks.rados.rados.0.smithi110.stdout:39: seq_num 38 ranges {0=638976} 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:39: writing smithi11075562-39 from 0 to 638976 tid 1 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:24: finishing write tid 2 to smithi11075562-24 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:24: finishing write tid 3 to smithi11075562-24 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:25: finishing write tid 1 to smithi11075562-25 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:24: done (15 left) 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:40: write initial oid 40 2024-09-14T03:37:42.115 INFO:tasks.rados.rados.0.smithi110.stdout:40: seq_num 39 ranges {0=663552} 2024-09-14T03:37:42.118 INFO:tasks.rados.rados.0.smithi110.stdout:40: writing smithi11075562-40 from 0 to 663552 tid 1 2024-09-14T03:37:42.118 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.118 INFO:tasks.rados.rados.0.smithi110.stdout:25: finishing write tid 2 to smithi11075562-25 2024-09-14T03:37:42.118 INFO:tasks.rados.rados.0.smithi110.stdout:25: finishing write tid 3 to smithi11075562-25 2024-09-14T03:37:42.118 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 6 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:26: finishing write tid 1 to smithi11075562-26 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:26: finishing write tid 2 to smithi11075562-26 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:26: finishing write tid 3 to smithi11075562-26 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 2 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:27: finishing write tid 1 to smithi11075562-27 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:25: done (15 left) 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:26: done (14 left) 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:41: write initial oid 41 2024-09-14T03:37:42.119 INFO:tasks.rados.rados.0.smithi110.stdout:41: seq_num 40 ranges {0=679936} 2024-09-14T03:37:42.122 INFO:tasks.rados.rados.0.smithi110.stdout:41: writing smithi11075562-41 from 0 to 679936 tid 1 2024-09-14T03:37:42.122 INFO:tasks.rados.rados.0.smithi110.stdout:42: write initial oid 42 2024-09-14T03:37:42.122 INFO:tasks.rados.rados.0.smithi110.stdout:42: seq_num 41 ranges {0=696320} 2024-09-14T03:37:42.127 INFO:tasks.rados.rados.0.smithi110.stdout:42: writing smithi11075562-42 from 0 to 696320 tid 1 2024-09-14T03:37:42.127 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.127 INFO:tasks.rados.rados.0.smithi110.stdout:27: finishing write tid 2 to smithi11075562-27 2024-09-14T03:37:42.127 INFO:tasks.rados.rados.0.smithi110.stdout:27: finishing write tid 3 to smithi11075562-27 2024-09-14T03:37:42.127 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 2 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-14T03:37:42.127 INFO:tasks.rados.rados.0.smithi110.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-09-14T03:37:42.127 INFO:tasks.rados.rados.0.smithi110.stdout:28: finishing write tid 1 to smithi11075562-28 2024-09-14T03:37:42.127 INFO:tasks.rados.rados.0.smithi110.stdout:28: finishing write tid 2 to smithi11075562-28 2024-09-14T03:37:42.128 INFO:tasks.rados.rados.0.smithi110.stdout:28: finishing write tid 3 to smithi11075562-28 2024-09-14T03:37:42.128 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 6 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-14T03:37:42.128 INFO:tasks.rados.rados.0.smithi110.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-09-14T03:37:42.128 INFO:tasks.rados.rados.0.smithi110.stdout:27: done (15 left) 2024-09-14T03:37:42.128 INFO:tasks.rados.rados.0.smithi110.stdout:28: done (14 left) 2024-09-14T03:37:42.128 INFO:tasks.rados.rados.0.smithi110.stdout:43: write initial oid 43 2024-09-14T03:37:42.128 INFO:tasks.rados.rados.0.smithi110.stdout:43: seq_num 42 ranges {0=712704} 2024-09-14T03:37:42.131 INFO:tasks.rados.rados.0.smithi110.stdout:43: writing smithi11075562-43 from 0 to 712704 tid 1 2024-09-14T03:37:42.131 INFO:tasks.rados.rados.0.smithi110.stdout:44: write initial oid 44 2024-09-14T03:37:42.131 INFO:tasks.rados.rados.0.smithi110.stdout:44: seq_num 43 ranges {0=729088} 2024-09-14T03:37:42.137 INFO:tasks.rados.rados.0.smithi110.stdout:44: writing smithi11075562-44 from 0 to 729088 tid 1 2024-09-14T03:37:42.137 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.137 INFO:tasks.rados.rados.0.smithi110.stdout:29: finishing write tid 1 to smithi11075562-29 2024-09-14T03:37:42.137 INFO:tasks.rados.rados.0.smithi110.stdout:29: finishing write tid 2 to smithi11075562-29 2024-09-14T03:37:42.137 INFO:tasks.rados.rados.0.smithi110.stdout:29: finishing write tid 3 to smithi11075562-29 2024-09-14T03:37:42.137 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 4 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-09-14T03:37:42.138 INFO:tasks.rados.rados.0.smithi110.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-09-14T03:37:42.138 INFO:tasks.rados.rados.0.smithi110.stdout:30: finishing write tid 1 to smithi11075562-30 2024-09-14T03:37:42.138 INFO:tasks.rados.rados.0.smithi110.stdout:30: finishing write tid 2 to smithi11075562-30 2024-09-14T03:37:42.138 INFO:tasks.rados.rados.0.smithi110.stdout:29: done (15 left) 2024-09-14T03:37:42.138 INFO:tasks.rados.rados.0.smithi110.stdout:45: write initial oid 45 2024-09-14T03:37:42.138 INFO:tasks.rados.rados.0.smithi110.stdout:45: seq_num 44 ranges {0=745472} 2024-09-14T03:37:42.142 INFO:tasks.rados.rados.0.smithi110.stdout:45: writing smithi11075562-45 from 0 to 745472 tid 1 2024-09-14T03:37:42.142 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.143 INFO:tasks.rados.rados.0.smithi110.stdout:30: finishing write tid 3 to smithi11075562-30 2024-09-14T03:37:42.143 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 8 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:37:42.143 INFO:tasks.rados.rados.0.smithi110.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:37:42.143 INFO:tasks.rados.rados.0.smithi110.stdout:31: finishing write tid 1 to smithi11075562-31 2024-09-14T03:37:42.143 INFO:tasks.rados.rados.0.smithi110.stdout:31: finishing write tid 2 to smithi11075562-31 2024-09-14T03:37:42.143 INFO:tasks.rados.rados.0.smithi110.stdout:30: done (15 left) 2024-09-14T03:37:42.143 INFO:tasks.rados.rados.0.smithi110.stdout:46: write initial oid 46 2024-09-14T03:37:42.143 INFO:tasks.rados.rados.0.smithi110.stdout:46: seq_num 45 ranges {0=761856} 2024-09-14T03:37:42.148 INFO:tasks.rados.rados.0.smithi110.stdout:46: writing smithi11075562-46 from 0 to 761856 tid 1 2024-09-14T03:37:42.148 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.148 INFO:tasks.rados.rados.0.smithi110.stdout:31: finishing write tid 3 to smithi11075562-31 2024-09-14T03:37:42.148 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 4 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-14T03:37:42.148 INFO:tasks.rados.rados.0.smithi110.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-09-14T03:37:42.148 INFO:tasks.rados.rados.0.smithi110.stdout:32: finishing write tid 1 to smithi11075562-32 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:32: finishing write tid 2 to smithi11075562-32 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:32: finishing write tid 3 to smithi11075562-32 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 10 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:33: finishing write tid 1 to smithi11075562-33 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:33: finishing write tid 2 to smithi11075562-33 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:33: finishing write tid 3 to smithi11075562-33 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 6 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:34: finishing write tid 1 to smithi11075562-34 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:34: finishing write tid 2 to smithi11075562-34 2024-09-14T03:37:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:34: finishing write tid 3 to smithi11075562-34 2024-09-14T03:37:42.150 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-14T03:37:42.150 INFO:tasks.rados.rados.0.smithi110.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-09-14T03:37:42.150 INFO:tasks.rados.rados.0.smithi110.stdout:31: done (15 left) 2024-09-14T03:37:42.150 INFO:tasks.rados.rados.0.smithi110.stdout:32: done (14 left) 2024-09-14T03:37:42.150 INFO:tasks.rados.rados.0.smithi110.stdout:33: done (13 left) 2024-09-14T03:37:42.150 INFO:tasks.rados.rados.0.smithi110.stdout:34: done (12 left) 2024-09-14T03:37:42.150 INFO:tasks.rados.rados.0.smithi110.stdout:47: write initial oid 47 2024-09-14T03:37:42.150 INFO:tasks.rados.rados.0.smithi110.stdout:47: seq_num 46 ranges {0=778240} 2024-09-14T03:37:42.152 INFO:tasks.rados.rados.0.smithi110.stdout:47: writing smithi11075562-47 from 0 to 778240 tid 1 2024-09-14T03:37:42.152 INFO:tasks.rados.rados.0.smithi110.stdout:48: write initial oid 48 2024-09-14T03:37:42.152 INFO:tasks.rados.rados.0.smithi110.stdout:48: seq_num 47 ranges {0=794624} 2024-09-14T03:37:42.157 INFO:tasks.rados.rados.0.smithi110.stdout:48: writing smithi11075562-48 from 0 to 794624 tid 1 2024-09-14T03:37:42.157 INFO:tasks.rados.rados.0.smithi110.stdout:49: write initial oid 49 2024-09-14T03:37:42.157 INFO:tasks.rados.rados.0.smithi110.stdout:49: seq_num 48 ranges {0=409600,409600=401408} 2024-09-14T03:37:42.159 INFO:tasks.rados.rados.0.smithi110.stdout:49: writing smithi11075562-49 from 0 to 409600 tid 1 2024-09-14T03:37:42.162 INFO:tasks.rados.rados.0.smithi110.stdout:49: writing smithi11075562-49 from 409600 to 811008 tid 2 2024-09-14T03:37:42.162 INFO:tasks.rados.rados.0.smithi110.stdout:50: write initial oid 50 2024-09-14T03:37:42.162 INFO:tasks.rados.rados.0.smithi110.stdout:50: seq_num 49 ranges {0=425984,425984=401408} 2024-09-14T03:37:42.164 INFO:tasks.rados.rados.0.smithi110.stdout:50: writing smithi11075562-50 from 0 to 425984 tid 1 2024-09-14T03:37:42.168 INFO:tasks.rados.rados.0.smithi110.stdout:50: writing smithi11075562-50 from 425984 to 827392 tid 2 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 16 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:35: finishing write tid 1 to smithi11075562-35 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:35: finishing write tid 2 to smithi11075562-35 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:35: finishing write tid 3 to smithi11075562-35 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 6 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:36: finishing write tid 1 to smithi11075562-36 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:36: finishing write tid 2 to smithi11075562-36 2024-09-14T03:37:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:36: finishing write tid 3 to smithi11075562-36 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 8 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:37: finishing write tid 1 to smithi11075562-37 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:35: done (15 left) 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:36: done (14 left) 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:51: copy_from oid 8 from oid 18 current snap is 0 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:37: finishing write tid 2 to smithi11075562-37 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:37: finishing write tid 3 to smithi11075562-37 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 10 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-09-14T03:37:42.170 INFO:tasks.rados.rados.0.smithi110.stdout:37: done (14 left) 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 0 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:38: finishing write tid 1 to smithi11075562-38 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:38: finishing write tid 2 to smithi11075562-38 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:38: finishing write tid 3 to smithi11075562-38 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 6 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:39: finishing write tid 1 to smithi11075562-39 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:39: finishing write tid 2 to smithi11075562-39 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:39: finishing write tid 3 to smithi11075562-39 2024-09-14T03:37:42.171 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 6 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:37:42.172 INFO:tasks.rados.rados.0.smithi110.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:37:42.172 INFO:tasks.rados.rados.0.smithi110.stdout:40: finishing write tid 1 to smithi11075562-40 2024-09-14T03:37:42.172 INFO:tasks.rados.rados.0.smithi110.stdout:52: seq_num 50 ranges {270336=442368,712704=401408} 2024-09-14T03:37:42.172 INFO:tasks.rados.rados.0.smithi110.stdout:52: writing smithi11075562-17 from 270336 to 712704 tid 1 2024-09-14T03:37:42.177 INFO:tasks.rados.rados.0.smithi110.stdout:52: writing smithi11075562-17 from 712704 to 1114112 tid 2 2024-09-14T03:37:42.177 INFO:tasks.rados.rados.0.smithi110.stdout:38: done (14 left) 2024-09-14T03:37:42.177 INFO:tasks.rados.rados.0.smithi110.stdout:39: done (13 left) 2024-09-14T03:37:42.177 INFO:tasks.rados.rados.0.smithi110.stdout:53: rmattr oid 20 current snap is 0 2024-09-14T03:37:42.177 INFO:tasks.rados.rados.0.smithi110.stdout:53: done (13 left) 2024-09-14T03:37:42.177 INFO:tasks.rados.rados.0.smithi110.stdout:54: delete oid 10 current snap is 0 2024-09-14T03:37:42.177 INFO:tasks.rados.rados.0.smithi110.stdout:40: finishing write tid 2 to smithi11075562-40 2024-09-14T03:37:42.178 INFO:tasks.rados.rados.0.smithi110.stdout:40: finishing write tid 3 to smithi11075562-40 2024-09-14T03:37:42.178 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 10 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-14T03:37:42.178 INFO:tasks.rados.rados.0.smithi110.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-09-14T03:37:42.178 INFO:tasks.rados.rados.0.smithi110.stdout:41: finishing write tid 1 to smithi11075562-41 2024-09-14T03:37:42.178 INFO:tasks.rados.rados.0.smithi110.stdout:41: finishing write tid 2 to smithi11075562-41 2024-09-14T03:37:42.178 INFO:tasks.rados.rados.0.smithi110.stdout:41: finishing write tid 3 to smithi11075562-41 2024-09-14T03:37:42.178 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 8 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-14T03:37:42.178 INFO:tasks.rados.rados.0.smithi110.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:42: finishing write tid 1 to smithi11075562-42 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:42: finishing write tid 2 to smithi11075562-42 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:42: finishing write tid 3 to smithi11075562-42 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 8 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:43: finishing write tid 1 to smithi11075562-43 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:43: finishing write tid 2 to smithi11075562-43 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:43: finishing write tid 3 to smithi11075562-43 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 10 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:44: finishing write tid 1 to smithi11075562-44 2024-09-14T03:37:42.179 INFO:tasks.rados.rados.0.smithi110.stdout:44: finishing write tid 2 to smithi11075562-44 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:44: finishing write tid 3 to smithi11075562-44 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 2 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:45: finishing write tid 1 to smithi11075562-45 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:45: finishing write tid 2 to smithi11075562-45 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:45: finishing write tid 3 to smithi11075562-45 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 12 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:46: finishing write tid 1 to smithi11075562-46 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:46: finishing write tid 2 to smithi11075562-46 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:46: finishing write tid 3 to smithi11075562-46 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 8 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:37:42.180 INFO:tasks.rados.rados.0.smithi110.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:47: finishing write tid 1 to smithi11075562-47 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:47: finishing write tid 2 to smithi11075562-47 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:47: finishing write tid 3 to smithi11075562-47 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 14 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:48: finishing write tid 1 to smithi11075562-48 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:48: finishing write tid 2 to smithi11075562-48 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:48: finishing write tid 3 to smithi11075562-48 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 6 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:49: finishing write tid 1 to smithi11075562-49 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:49: finishing write tid 2 to smithi11075562-49 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:49: finishing write tid 3 to smithi11075562-49 2024-09-14T03:37:42.181 INFO:tasks.rados.rados.0.smithi110.stdout:49: finishing write tid 4 to smithi11075562-49 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 11 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:50: finishing write tid 1 to smithi11075562-50 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:50: finishing write tid 2 to smithi11075562-50 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:50: finishing write tid 3 to smithi11075562-50 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:50: finishing write tid 4 to smithi11075562-50 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 3 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:51: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:51: finishing copy_from to smithi11075562-8 2024-09-14T03:37:42.182 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 6 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:52: finishing write tid 1 to smithi11075562-17 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:40: done (13 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:41: done (12 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:42: done (11 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:43: done (10 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:44: done (9 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:45: done (8 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:46: done (7 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:47: done (6 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:48: done (5 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:49: done (4 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:50: done (3 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:51: done (2 left) 2024-09-14T03:37:42.183 INFO:tasks.rados.rados.0.smithi110.stdout:54: done (1 left) 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:55: read oid 15 snap -1 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:56: rollback oid 31 current snap is 0 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:56: done (2 left) 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:57: read oid 28 snap -1 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:58: copy_from oid 29 from oid 47 current snap is 0 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:59: rollback oid 45 current snap is 0 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:59: done (4 left) 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:60: copy_from oid 23 from oid 48 current snap is 0 2024-09-14T03:37:42.184 INFO:tasks.rados.rados.0.smithi110.stdout:61: delete oid 1 current snap is 0 2024-09-14T03:37:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:52: finishing write tid 2 to smithi11075562-17 2024-09-14T03:37:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:52: finishing write tid 3 to smithi11075562-17 2024-09-14T03:37:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:61: done (5 left) 2024-09-14T03:37:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:62: read oid 18 snap -1 2024-09-14T03:37:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:62: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-14T03:37:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:63: snap_create 2024-09-14T03:37:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:52: finishing write tid 4 to smithi11075562-17 2024-09-14T03:37:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 9 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-14T03:37:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:52: left oid 17 (ObjNum 50 snap 0 seq_num 50) 2024-09-14T03:37:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-14T03:37:42.194 INFO:tasks.rados.rados.0.smithi110.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-14T03:37:42.201 INFO:tasks.rados.rados.0.smithi110.stdout:62: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-14T03:37:42.212 INFO:tasks.rados.rados.0.smithi110.stdout:58: finishing copy_from racing read to smithi11075562-29 2024-09-14T03:37:42.213 INFO:tasks.rados.rados.0.smithi110.stdout:60: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:37:42.213 INFO:tasks.rados.rados.0.smithi110.stdout:58: finishing copy_from to smithi11075562-29 2024-09-14T03:37:42.213 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 14 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:37:42.213 INFO:tasks.rados.rados.0.smithi110.stdout:60: finishing copy_from to smithi11075562-23 2024-09-14T03:37:42.213 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 6 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-14T03:37:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:42 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2946418321' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:37:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:42 smithi110 ceph-mon[35374]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:37:42.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:42 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2946418321' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:37:42.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:42 smithi110 ceph-mon[27365]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:37:42.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:42 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2946418321' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:37:42.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:42 smithi195 ceph-mon[30695]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:37:43.034 INFO:tasks.rados.rados.0.smithi110.stdout:52: done (6 left) 2024-09-14T03:37:43.034 INFO:tasks.rados.rados.0.smithi110.stdout:55: done (5 left) 2024-09-14T03:37:43.034 INFO:tasks.rados.rados.0.smithi110.stdout:57: done (4 left) 2024-09-14T03:37:43.034 INFO:tasks.rados.rados.0.smithi110.stdout:58: done (3 left) 2024-09-14T03:37:43.034 INFO:tasks.rados.rados.0.smithi110.stdout:60: done (2 left) 2024-09-14T03:37:43.035 INFO:tasks.rados.rados.0.smithi110.stdout:62: done (1 left) 2024-09-14T03:37:43.035 INFO:tasks.rados.rados.0.smithi110.stdout:63: done (0 left) 2024-09-14T03:37:43.035 INFO:tasks.rados.rados.0.smithi110.stdout:64: copy_from oid 15 from oid 2 current snap is 1 2024-09-14T03:37:43.035 INFO:tasks.rados.rados.0.smithi110.stdout:65: read oid 30 snap -1 2024-09-14T03:37:43.035 INFO:tasks.rados.rados.0.smithi110.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:37:43.035 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 1 2024-09-14T03:37:43.035 INFO:tasks.rados.rados.0.smithi110.stdout:66: seq_num 53 ranges {794624=491520,1286144=401408} 2024-09-14T03:37:43.041 INFO:tasks.rados.rados.0.smithi110.stdout:66: writing smithi11075562-23 from 794624 to 1286144 tid 1 2024-09-14T03:37:43.044 INFO:tasks.rados.rados.0.smithi110.stdout:66: writing smithi11075562-23 from 1286144 to 1687552 tid 2 2024-09-14T03:37:43.044 INFO:tasks.rados.rados.0.smithi110.stdout:67: rollback oid 10 current snap is 1 2024-09-14T03:37:43.044 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 0 2024-09-14T03:37:43.045 INFO:tasks.rados.rados.0.smithi110.stdout:64: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:37:43.045 INFO:tasks.rados.rados.0.smithi110.stdout:68: setattr oid 32 current snap is 1 2024-09-14T03:37:43.045 INFO:tasks.rados.rados.0.smithi110.stdout:64: finishing copy_from to smithi11075562-15 2024-09-14T03:37:43.045 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-14T03:37:43.045 INFO:tasks.rados.rados.0.smithi110.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:66: finishing write tid 1 to smithi11075562-23 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:66: finishing write tid 2 to smithi11075562-23 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:66: finishing write tid 3 to smithi11075562-23 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:67: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 0 (ObjNum 64643082 snap 21934 seq_num 304) dirty dne 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:66: finishing write tid 4 to smithi11075562-23 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 12 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:66: left oid 23 (ObjNum 53 snap 1 seq_num 53) 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:64: done (4 left) 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:65: done (3 left) 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:66: done (2 left) 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:67: done (1 left) 2024-09-14T03:37:43.061 INFO:tasks.rados.rados.0.smithi110.stdout:69: copy_from oid 45 from oid 34 current snap is 1 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:70: read oid 23 snap 0 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:70: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:71: copy_from oid 36 from oid 27 current snap is 1 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:72: rollback oid 49 current snap is 1 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 0 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:73: read oid 4 snap 0 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:74: rmattr oid 7 current snap is 1 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:74: done (6 left) 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:75: copy_from oid 20 from oid 12 current snap is 1 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:76: read oid 39 snap -1 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:37:43.062 INFO:tasks.rados.rados.0.smithi110.stdout:77: copy_from oid 6 from oid 37 current snap is 1 2024-09-14T03:37:43.063 INFO:tasks.rados.rados.0.smithi110.stdout:78: read oid 2 snap -1 2024-09-14T03:37:43.063 INFO:tasks.rados.rados.0.smithi110.stdout:78: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:37:43.063 INFO:tasks.rados.rados.0.smithi110.stdout:79: setattr oid 41 current snap is 1 2024-09-14T03:37:43.063 INFO:tasks.rados.rados.0.smithi110.stdout:80: snap_create 2024-09-14T03:37:43.064 INFO:tasks.rados.rados.0.smithi110.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:37:43.067 INFO:tasks.rados.rados.0.smithi110.stdout:78: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:37:43.068 INFO:tasks.rados.rados.0.smithi110.stdout:72: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:37:43.069 INFO:tasks.rados.rados.0.smithi110.stdout:70: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-14T03:37:43.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 15 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-14T03:37:43.093 INFO:tasks.rados.rados.0.smithi110.stdout:72: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:37:43.094 INFO:tasks.rados.rados.0.smithi110.stdout:72: finishing rollback tid 2 to smithi11075562-49 2024-09-14T03:37:43.094 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 16 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-14T03:37:43.094 INFO:tasks.rados.rados.0.smithi110.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:37:43.113 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 12 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-14T03:37:43.113 INFO:tasks.rados.rados.0.smithi110.stdout:75: finishing copy_from racing read to smithi11075562-20 2024-09-14T03:37:43.113 INFO:tasks.rados.rados.0.smithi110.stdout:77: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:37:43.113 INFO:tasks.rados.rados.0.smithi110.stdout:75: finishing copy_from to smithi11075562-20 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 4 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:77: finishing copy_from to smithi11075562-6 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 10 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:71: finishing copy_from racing read to smithi11075562-36 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:69: finishing copy_from to smithi11075562-45 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:69: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:71: finishing copy_from to smithi11075562-36 2024-09-14T03:37:43.114 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-14T03:37:43.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:43 smithi195 ceph-mon[30695]: pgmap v102: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:43.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:43 smithi195 ceph-mon[30695]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:37:43.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:43 smithi195 ceph-mon[30695]: Cluster is now healthy 2024-09-14T03:37:43.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:43 smithi195 ceph-mon[30695]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-14T03:37:43.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:43 smithi195 ceph-mon[30695]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:37:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[35374]: pgmap v102: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[35374]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:37:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[35374]: Cluster is now healthy 2024-09-14T03:37:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[35374]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-14T03:37:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[35374]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:37:43.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[27365]: pgmap v102: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:43.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[27365]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:37:43.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[27365]: Cluster is now healthy 2024-09-14T03:37:43.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[27365]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-14T03:37:43.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:43 smithi110 ceph-mon[27365]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:68: done (11 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:69: done (10 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:70: done (9 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:71: done (8 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:72: done (7 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:73: done (6 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:75: done (5 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:76: done (4 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:77: done (3 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:78: done (2 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:79: done (1 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:80: done (0 left) 2024-09-14T03:37:44.039 INFO:tasks.rados.rados.0.smithi110.stdout:81: rollback oid 40 current snap is 2 2024-09-14T03:37:44.040 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 40 to 1 2024-09-14T03:37:44.040 INFO:tasks.rados.rados.0.smithi110.stdout:82: snap_create 2024-09-14T03:37:44.041 INFO:tasks.rados.rados.0.smithi110.stdout:81: finishing rollback tid 0 to smithi11075562-40 2024-09-14T03:37:44.041 INFO:tasks.rados.rados.0.smithi110.stdout:81: finishing rollback tid 1 to smithi11075562-40 2024-09-14T03:37:44.041 INFO:tasks.rados.rados.0.smithi110.stdout:81: finishing rollback tid 2 to smithi11075562-40 2024-09-14T03:37:44.041 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 16 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-14T03:37:44.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:44 smithi110 ceph-mon[27365]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:37:44.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:44 smithi110 ceph-mon[35374]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:37:44.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:44 smithi195 ceph-mon[30695]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:37:45.043 INFO:tasks.rados.rados.0.smithi110.stdout:81: done (1 left) 2024-09-14T03:37:45.043 INFO:tasks.rados.rados.0.smithi110.stdout:82: done (0 left) 2024-09-14T03:37:45.043 INFO:tasks.rados.rados.0.smithi110.stdout:83: rollback oid 9 current snap is 3 2024-09-14T03:37:45.043 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 9 to 0 2024-09-14T03:37:45.043 INFO:tasks.rados.rados.0.smithi110.stdout:84: read oid 20 snap 1 2024-09-14T03:37:45.044 INFO:tasks.rados.rados.0.smithi110.stdout:84: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-14T03:37:45.044 INFO:tasks.rados.rados.0.smithi110.stdout:85: read oid 41 snap -1 2024-09-14T03:37:45.044 INFO:tasks.rados.rados.0.smithi110.stdout:85: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-14T03:37:45.044 INFO:tasks.rados.rados.0.smithi110.stdout:86: snap_create 2024-09-14T03:37:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:85: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-14T03:37:45.068 INFO:tasks.rados.rados.0.smithi110.stdout:83: finishing rollback tid 0 to smithi11075562-9 2024-09-14T03:37:45.068 INFO:tasks.rados.rados.0.smithi110.stdout:83: finishing rollback tid 1 to smithi11075562-9 2024-09-14T03:37:45.068 INFO:tasks.rados.rados.0.smithi110.stdout:83: finishing rollback tid 2 to smithi11075562-9 2024-09-14T03:37:45.068 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 14 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-14T03:37:45.069 INFO:tasks.rados.rados.0.smithi110.stdout:84: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-14T03:37:45.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:45 smithi110 ceph-mon[27365]: pgmap v105: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:45.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:45 smithi110 ceph-mon[27365]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:37:45.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:45 smithi110 ceph-mon[35374]: pgmap v105: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:45.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:45 smithi110 ceph-mon[35374]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:37:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:45 smithi195 ceph-mon[30695]: pgmap v105: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:45 smithi195 ceph-mon[30695]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:37:46.047 INFO:tasks.rados.rados.0.smithi110.stdout:83: done (3 left) 2024-09-14T03:37:46.048 INFO:tasks.rados.rados.0.smithi110.stdout:84: done (2 left) 2024-09-14T03:37:46.048 INFO:tasks.rados.rados.0.smithi110.stdout:85: done (1 left) 2024-09-14T03:37:46.048 INFO:tasks.rados.rados.0.smithi110.stdout:86: done (0 left) 2024-09-14T03:37:46.048 INFO:tasks.rados.rados.0.smithi110.stdout:87: snap_remove snap 0 2024-09-14T03:37:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:46 smithi110 ceph-mon[35374]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:37:46.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:46 smithi110 ceph-mon[27365]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:37:46.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:46 smithi195 ceph-mon[30695]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:37:47.051 INFO:tasks.rados.rados.0.smithi110.stdout:87: done (0 left) 2024-09-14T03:37:47.051 INFO:tasks.rados.rados.0.smithi110.stdout:88: read oid 29 snap 3 2024-09-14T03:37:47.051 INFO:tasks.rados.rados.0.smithi110.stdout:88: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:37:47.051 INFO:tasks.rados.rados.0.smithi110.stdout:89: read oid 46 snap -1 2024-09-14T03:37:47.052 INFO:tasks.rados.rados.0.smithi110.stdout:89: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:37:47.052 INFO:tasks.rados.rados.0.smithi110.stdout:90: copy_from oid 36 from oid 2 current snap is 4 2024-09-14T03:37:47.052 INFO:tasks.rados.rados.0.smithi110.stdout:91: rmattr oid 49 current snap is 4 2024-09-14T03:37:47.052 INFO:tasks.rados.rados.0.smithi110.stdout:91: done (3 left) 2024-09-14T03:37:47.052 INFO:tasks.rados.rados.0.smithi110.stdout:92: snap_remove snap 2 2024-09-14T03:37:47.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:47 smithi110 ceph-mon[35374]: pgmap v108: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:47.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:47 smithi110 ceph-mon[35374]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:37:47.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:47 smithi110 ceph-mon[27365]: pgmap v108: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:47.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:47 smithi110 ceph-mon[27365]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:37:47.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:47 smithi195 ceph-mon[30695]: pgmap v108: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:47.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:47 smithi195 ceph-mon[30695]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:37:48.060 INFO:tasks.rados.rados.0.smithi110.stdout:92: done (3 left) 2024-09-14T03:37:48.060 INFO:tasks.rados.rados.0.smithi110.stdout:93: rmattr oid 30 current snap is 4 2024-09-14T03:37:48.060 INFO:tasks.rados.rados.0.smithi110.stdout:93: done (3 left) 2024-09-14T03:37:48.060 INFO:tasks.rados.rados.0.smithi110.stdout:94: read oid 42 snap 1 2024-09-14T03:37:48.060 INFO:tasks.rados.rados.0.smithi110.stdout:94: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-14T03:37:48.060 INFO:tasks.rados.rados.0.smithi110.stdout:95: rmattr oid 14 current snap is 4 2024-09-14T03:37:48.060 INFO:tasks.rados.rados.0.smithi110.stdout:95: done (4 left) 2024-09-14T03:37:48.060 INFO:tasks.rados.rados.0.smithi110.stdout:96: read oid 9 snap 1 2024-09-14T03:37:48.061 INFO:tasks.rados.rados.0.smithi110.stdout:96: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-14T03:37:48.061 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 4 2024-09-14T03:37:48.061 INFO:tasks.rados.rados.0.smithi110.stdout:97: seq_num 54 ranges {393216=507904,901120=401408} 2024-09-14T03:37:48.065 INFO:tasks.rados.rados.0.smithi110.stdout:97: writing smithi11075562-24 from 393216 to 901120 tid 1 2024-09-14T03:37:48.068 INFO:tasks.rados.rados.0.smithi110.stdout:97: writing smithi11075562-24 from 901120 to 1302528 tid 2 2024-09-14T03:37:48.068 INFO:tasks.rados.rados.0.smithi110.stdout:98: copy_from oid 31 from oid 6 current snap is 4 2024-09-14T03:37:48.068 INFO:tasks.rados.rados.0.smithi110.stdout:88: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:37:48.091 INFO:tasks.rados.rados.0.smithi110.stdout:88: done (6 left) 2024-09-14T03:37:48.092 INFO:tasks.rados.rados.0.smithi110.stdout:99: snap_remove snap 3 2024-09-14T03:37:48.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:48 smithi110 ceph-mon[35374]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:37:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:48 smithi110 ceph-mon[27365]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:37:48.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:48 smithi195 ceph-mon[30695]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:37:49.063 INFO:tasks.rados.rados.0.smithi110.stdout:99: done (6 left) 2024-09-14T03:37:49.064 INFO:tasks.rados.rados.0.smithi110.stdout:100: copy_from oid 39 from oid 26 current snap is 4 2024-09-14T03:37:49.064 INFO:tasks.rados.rados.0.smithi110.stdout:101: read oid 50 snap 1 2024-09-14T03:37:49.064 INFO:tasks.rados.rados.0.smithi110.stdout:101: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:37:49.064 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 4 2024-09-14T03:37:49.064 INFO:tasks.rados.rados.0.smithi110.stdout:90: finishing copy_from racing read to smithi11075562-36 2024-09-14T03:37:49.064 INFO:tasks.rados.rados.0.smithi110.stdout:90: finishing copy_from to smithi11075562-36 2024-09-14T03:37:49.064 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-14T03:37:49.064 INFO:tasks.rados.rados.0.smithi110.stdout:89: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:37:49.090 INFO:tasks.rados.rados.0.smithi110.stdout:102: seq_num 55 ranges {73728=524288,598016=401408} 2024-09-14T03:37:49.093 INFO:tasks.rados.rados.0.smithi110.stdout:102: writing smithi11075562-5 from 73728 to 598016 tid 1 2024-09-14T03:37:49.096 INFO:tasks.rados.rados.0.smithi110.stdout:102: writing smithi11075562-5 from 598016 to 999424 tid 2 2024-09-14T03:37:49.096 INFO:tasks.rados.rados.0.smithi110.stdout:89: done (8 left) 2024-09-14T03:37:49.096 INFO:tasks.rados.rados.0.smithi110.stdout:90: done (7 left) 2024-09-14T03:37:49.096 INFO:tasks.rados.rados.0.smithi110.stdout:103: delete oid 45 current snap is 4 2024-09-14T03:37:49.096 INFO:tasks.rados.rados.0.smithi110.stdout:94: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-14T03:37:49.116 INFO:tasks.rados.rados.0.smithi110.stdout:96: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:97: finishing write tid 1 to smithi11075562-24 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:97: finishing write tid 2 to smithi11075562-24 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:97: finishing write tid 3 to smithi11075562-24 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:97: finishing write tid 4 to smithi11075562-24 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 19 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:97: left oid 24 (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:98: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:98: finishing copy_from to smithi11075562-31 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 10 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:37:49.121 INFO:tasks.rados.rados.0.smithi110.stdout:101: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:37:49.146 INFO:tasks.rados.rados.0.smithi110.stdout:100: finishing copy_from racing read to smithi11075562-39 2024-09-14T03:37:49.146 INFO:tasks.rados.rados.0.smithi110.stdout:100: finishing copy_from to smithi11075562-39 2024-09-14T03:37:49.146 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 2 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-14T03:37:49.146 INFO:tasks.rados.rados.0.smithi110.stdout:94: done (7 left) 2024-09-14T03:37:49.146 INFO:tasks.rados.rados.0.smithi110.stdout:96: done (6 left) 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:97: done (5 left) 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:98: done (4 left) 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:100: done (3 left) 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:101: done (2 left) 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:103: done (1 left) 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:104: read oid 2 snap -1 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:104: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:105: rollback oid 34 current snap is 4 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 1 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:102: finishing write tid 1 to smithi11075562-5 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:102: finishing write tid 2 to smithi11075562-5 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:102: finishing write tid 3 to smithi11075562-5 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:102: finishing write tid 4 to smithi11075562-5 2024-09-14T03:37:49.147 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 32 (ObjNum 55 snap 4 seq_num 55) dirty exists 2024-09-14T03:37:49.148 INFO:tasks.rados.rados.0.smithi110.stdout:102: left oid 5 (ObjNum 55 snap 4 seq_num 55) 2024-09-14T03:37:49.148 INFO:tasks.rados.rados.0.smithi110.stdout:102: done (2 left) 2024-09-14T03:37:49.148 INFO:tasks.rados.rados.0.smithi110.stdout:106: delete oid 12 current snap is 4 2024-09-14T03:37:49.148 INFO:tasks.rados.rados.0.smithi110.stdout:106: done (2 left) 2024-09-14T03:37:49.148 INFO:tasks.rados.rados.0.smithi110.stdout:107: read oid 17 snap 1 2024-09-14T03:37:49.148 INFO:tasks.rados.rados.0.smithi110.stdout:107: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-14T03:37:49.149 INFO:tasks.rados.rados.0.smithi110.stdout:108: rollback oid 41 current snap is 4 2024-09-14T03:37:49.149 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 1 2024-09-14T03:37:49.149 INFO:tasks.rados.rados.0.smithi110.stdout:109: delete oid 8 current snap is 4 2024-09-14T03:37:49.149 INFO:tasks.rados.rados.0.smithi110.stdout:104: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:37:49.150 INFO:tasks.rados.rados.0.smithi110.stdout:105: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:37:49.150 INFO:tasks.rados.rados.0.smithi110.stdout:105: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:37:49.150 INFO:tasks.rados.rados.0.smithi110.stdout:104: done (4 left) 2024-09-14T03:37:49.150 INFO:tasks.rados.rados.0.smithi110.stdout:109: done (3 left) 2024-09-14T03:37:49.150 INFO:tasks.rados.rados.0.smithi110.stdout:110: setattr oid 46 current snap is 4 2024-09-14T03:37:49.150 INFO:tasks.rados.rados.0.smithi110.stdout:108: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:37:49.151 INFO:tasks.rados.rados.0.smithi110.stdout:105: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:37:49.151 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 14 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-14T03:37:49.151 INFO:tasks.rados.rados.0.smithi110.stdout:105: done (3 left) 2024-09-14T03:37:49.151 INFO:tasks.rados.rados.0.smithi110.stdout:111: snap_create 2024-09-14T03:37:49.151 INFO:tasks.rados.rados.0.smithi110.stdout:108: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:37:49.151 INFO:tasks.rados.rados.0.smithi110.stdout:108: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:37:49.151 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 20 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-14T03:37:49.153 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 22 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:37:49.156 INFO:tasks.rados.rados.0.smithi110.stdout:107: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-14T03:37:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:49 smithi110 ceph-mon[35374]: pgmap v111: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 6.3 MiB/s wr, 78 op/s 2024-09-14T03:37:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:49 smithi110 ceph-mon[35374]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:37:49.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:49 smithi110 ceph-mon[27365]: pgmap v111: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 6.3 MiB/s wr, 78 op/s 2024-09-14T03:37:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:49 smithi110 ceph-mon[27365]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:37:49.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:49 smithi195 ceph-mon[30695]: pgmap v111: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 6.3 MiB/s wr, 78 op/s 2024-09-14T03:37:49.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:49 smithi195 ceph-mon[30695]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:37:50.068 INFO:tasks.rados.rados.0.smithi110.stdout:107: done (3 left) 2024-09-14T03:37:50.068 INFO:tasks.rados.rados.0.smithi110.stdout:108: done (2 left) 2024-09-14T03:37:50.069 INFO:tasks.rados.rados.0.smithi110.stdout:110: done (1 left) 2024-09-14T03:37:50.069 INFO:tasks.rados.rados.0.smithi110.stdout:111: done (0 left) 2024-09-14T03:37:50.069 INFO:tasks.rados.rados.0.smithi110.stdout:112: copy_from oid 25 from oid 6 current snap is 5 2024-09-14T03:37:50.069 INFO:tasks.rados.rados.0.smithi110.stdout:113: setattr oid 23 current snap is 5 2024-09-14T03:37:50.069 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 5 2024-09-14T03:37:50.069 INFO:tasks.rados.rados.0.smithi110.stdout:114: seq_num 59 ranges {24576=598016,622592=401408} 2024-09-14T03:37:50.073 INFO:tasks.rados.rados.0.smithi110.stdout:114: writing smithi11075562-15 from 24576 to 622592 tid 1 2024-09-14T03:37:50.076 INFO:tasks.rados.rados.0.smithi110.stdout:114: writing smithi11075562-15 from 622592 to 1024000 tid 2 2024-09-14T03:37:50.076 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 5 2024-09-14T03:37:50.076 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 23 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-14T03:37:50.076 INFO:tasks.rados.rados.0.smithi110.stdout:112: finishing copy_from racing read to smithi11075562-25 2024-09-14T03:37:50.076 INFO:tasks.rados.rados.0.smithi110.stdout:115: seq_num 60 ranges {188416=614400,802816=401408} 2024-09-14T03:37:50.081 INFO:tasks.rados.rados.0.smithi110.stdout:115: writing smithi11075562-20 from 188416 to 802816 tid 1 2024-09-14T03:37:50.084 INFO:tasks.rados.rados.0.smithi110.stdout:115: writing smithi11075562-20 from 802816 to 1204224 tid 2 2024-09-14T03:37:50.084 INFO:tasks.rados.rados.0.smithi110.stdout:113: done (3 left) 2024-09-14T03:37:50.084 INFO:tasks.rados.rados.0.smithi110.stdout:116: rollback oid 14 current snap is 5 2024-09-14T03:37:50.084 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 14 to 1 2024-09-14T03:37:50.084 INFO:tasks.rados.rados.0.smithi110.stdout:112: finishing copy_from to smithi11075562-25 2024-09-14T03:37:50.084 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 10 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:37:50.085 INFO:tasks.rados.rados.0.smithi110.stdout:114: finishing write tid 1 to smithi11075562-15 2024-09-14T03:37:50.085 INFO:tasks.rados.rados.0.smithi110.stdout:114: finishing write tid 2 to smithi11075562-15 2024-09-14T03:37:50.085 INFO:tasks.rados.rados.0.smithi110.stdout:112: done (3 left) 2024-09-14T03:37:50.085 INFO:tasks.rados.rados.0.smithi110.stdout:117: read oid 22 snap -1 2024-09-14T03:37:50.085 INFO:tasks.rados.rados.0.smithi110.stdout:117: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-14T03:37:50.085 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 5 2024-09-14T03:37:50.085 INFO:tasks.rados.rados.0.smithi110.stdout:118: seq_num 61 ranges {663552=630784,1294336=401408} 2024-09-14T03:37:50.091 INFO:tasks.rados.rados.0.smithi110.stdout:118: writing smithi11075562-40 from 663552 to 1294336 tid 1 2024-09-14T03:37:50.094 INFO:tasks.rados.rados.0.smithi110.stdout:118: writing smithi11075562-40 from 1294336 to 1695744 tid 2 2024-09-14T03:37:50.094 INFO:tasks.rados.rados.0.smithi110.stdout:119: snap_create 2024-09-14T03:37:50.095 INFO:tasks.rados.rados.0.smithi110.stdout:114: finishing write tid 3 to smithi11075562-15 2024-09-14T03:37:50.095 INFO:tasks.rados.rados.0.smithi110.stdout:114: finishing write tid 4 to smithi11075562-15 2024-09-14T03:37:50.095 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 25 (ObjNum 59 snap 5 seq_num 59) dirty exists 2024-09-14T03:37:50.095 INFO:tasks.rados.rados.0.smithi110.stdout:114: left oid 15 (ObjNum 59 snap 5 seq_num 59) 2024-09-14T03:37:50.095 INFO:tasks.rados.rados.0.smithi110.stdout:117: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-14T03:37:50.104 INFO:tasks.rados.rados.0.smithi110.stdout:115: finishing write tid 1 to smithi11075562-20 2024-09-14T03:37:50.104 INFO:tasks.rados.rados.0.smithi110.stdout:115: finishing write tid 2 to smithi11075562-20 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:115: finishing write tid 3 to smithi11075562-20 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:116: finishing rollback tid 0 to smithi11075562-14 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:116: finishing rollback tid 1 to smithi11075562-14 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:115: finishing write tid 4 to smithi11075562-20 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 26 (ObjNum 60 snap 5 seq_num 60) dirty exists 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:115: left oid 20 (ObjNum 60 snap 5 seq_num 60) 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:116: finishing rollback tid 2 to smithi11075562-14 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 24 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:118: finishing write tid 1 to smithi11075562-40 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:118: finishing write tid 2 to smithi11075562-40 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:118: finishing write tid 3 to smithi11075562-40 2024-09-14T03:37:50.105 INFO:tasks.rados.rados.0.smithi110.stdout:118: finishing write tid 4 to smithi11075562-40 2024-09-14T03:37:50.106 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 24 (ObjNum 61 snap 5 seq_num 61) dirty exists 2024-09-14T03:37:50.106 INFO:tasks.rados.rados.0.smithi110.stdout:118: left oid 40 (ObjNum 61 snap 5 seq_num 61) 2024-09-14T03:37:50.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:50 smithi110 ceph-mon[35374]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:37:50.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:50 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:50.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:50 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:50.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:50 smithi110 ceph-mon[27365]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:37:50.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:50 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:50.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:50 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:50.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:37:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:37:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:50 smithi195 ceph-mon[30695]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:37:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:50 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:37:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:50 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:37:50.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:37:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:37:50] "GET /metrics HTTP/1.1" 200 239649 "" "Prometheus/2.43.0" 2024-09-14T03:37:51.073 INFO:tasks.rados.rados.0.smithi110.stdout:114: done (5 left) 2024-09-14T03:37:51.074 INFO:tasks.rados.rados.0.smithi110.stdout:115: done (4 left) 2024-09-14T03:37:51.074 INFO:tasks.rados.rados.0.smithi110.stdout:116: done (3 left) 2024-09-14T03:37:51.074 INFO:tasks.rados.rados.0.smithi110.stdout:117: done (2 left) 2024-09-14T03:37:51.074 INFO:tasks.rados.rados.0.smithi110.stdout:118: done (1 left) 2024-09-14T03:37:51.074 INFO:tasks.rados.rados.0.smithi110.stdout:119: done (0 left) 2024-09-14T03:37:51.074 INFO:tasks.rados.rados.0.smithi110.stdout:120: read oid 48 snap 4 2024-09-14T03:37:51.075 INFO:tasks.rados.rados.0.smithi110.stdout:120: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-14T03:37:51.075 INFO:tasks.rados.rados.0.smithi110.stdout:121: read oid 49 snap 5 2024-09-14T03:37:51.075 INFO:tasks.rados.rados.0.smithi110.stdout:121: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-14T03:37:51.075 INFO:tasks.rados.rados.0.smithi110.stdout:122: delete oid 44 current snap is 6 2024-09-14T03:37:51.079 INFO:tasks.rados.rados.0.smithi110.stdout:122: done (2 left) 2024-09-14T03:37:51.079 INFO:tasks.rados.rados.0.smithi110.stdout:123: rollback oid 23 current snap is 6 2024-09-14T03:37:51.079 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 23 to 4 2024-09-14T03:37:51.079 INFO:tasks.rados.rados.0.smithi110.stdout:124: read oid 30 snap -1 2024-09-14T03:37:51.079 INFO:tasks.rados.rados.0.smithi110.stdout:124: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:37:51.079 INFO:tasks.rados.rados.0.smithi110.stdout:125: setattr oid 35 current snap is 6 2024-09-14T03:37:51.080 INFO:tasks.rados.rados.0.smithi110.stdout:120: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-14T03:37:51.103 INFO:tasks.rados.rados.0.smithi110.stdout:123: finishing rollback tid 0 to smithi11075562-23 2024-09-14T03:37:51.103 INFO:tasks.rados.rados.0.smithi110.stdout:124: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:37:51.116 INFO:tasks.rados.rados.0.smithi110.stdout:123: finishing rollback tid 1 to smithi11075562-23 2024-09-14T03:37:51.117 INFO:tasks.rados.rados.0.smithi110.stdout:123: finishing rollback tid 2 to smithi11075562-23 2024-09-14T03:37:51.117 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 27 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-14T03:37:51.117 INFO:tasks.rados.rados.0.smithi110.stdout:121: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-14T03:37:51.137 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 20 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-14T03:37:51.138 INFO:tasks.rados.rados.0.smithi110.stdout:120: done (4 left) 2024-09-14T03:37:51.138 INFO:tasks.rados.rados.0.smithi110.stdout:121: done (3 left) 2024-09-14T03:37:51.138 INFO:tasks.rados.rados.0.smithi110.stdout:123: done (2 left) 2024-09-14T03:37:51.138 INFO:tasks.rados.rados.0.smithi110.stdout:124: done (1 left) 2024-09-14T03:37:51.139 INFO:tasks.rados.rados.0.smithi110.stdout:125: done (0 left) 2024-09-14T03:37:51.139 INFO:tasks.rados.rados.0.smithi110.stdout:126: delete oid 10 current snap is 6 2024-09-14T03:37:51.139 INFO:tasks.rados.rados.0.smithi110.stdout:126: done (0 left) 2024-09-14T03:37:51.139 INFO:tasks.rados.rados.0.smithi110.stdout:127: snap_remove snap 4 2024-09-14T03:37:51.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:51 smithi110 ceph-mon[35374]: pgmap v114: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 6.3 MiB/s wr, 78 op/s 2024-09-14T03:37:51.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:51 smithi110 ceph-mon[35374]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:37:51.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:51 smithi110 ceph-mon[27365]: pgmap v114: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 6.3 MiB/s wr, 78 op/s 2024-09-14T03:37:51.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:51 smithi110 ceph-mon[27365]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:37:51.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:51 smithi195 ceph-mon[30695]: pgmap v114: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 6.3 MiB/s wr, 78 op/s 2024-09-14T03:37:51.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:51 smithi195 ceph-mon[30695]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:37:52.078 INFO:tasks.rados.rados.0.smithi110.stdout:127: done (0 left) 2024-09-14T03:37:52.079 INFO:tasks.rados.rados.0.smithi110.stdout:128: setattr oid 28 current snap is 6 2024-09-14T03:37:52.081 INFO:tasks.rados.rados.0.smithi110.stdout:129: snap_remove snap 1 2024-09-14T03:37:52.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:52 smithi110 ceph-mon[35374]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:37:52.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:52 smithi110 ceph-mon[27365]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:37:52.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:52 smithi195 ceph-mon[30695]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:37:53.079 INFO:tasks.rados.rados.0.smithi110.stdout:129: done (1 left) 2024-09-14T03:37:53.079 INFO:tasks.rados.rados.0.smithi110.stdout:130: delete oid 6 current snap is 6 2024-09-14T03:37:53.079 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 8 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-14T03:37:53.085 INFO:tasks.rados.rados.0.smithi110.stdout:128: done (1 left) 2024-09-14T03:37:53.085 INFO:tasks.rados.rados.0.smithi110.stdout:130: done (0 left) 2024-09-14T03:37:53.085 INFO:tasks.rados.rados.0.smithi110.stdout:131: snap_remove snap 5 2024-09-14T03:37:53.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:53 smithi110 ceph-mon[35374]: pgmap v117: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-09-14T03:37:53.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:53 smithi110 ceph-mon[35374]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:37:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:53 smithi110 ceph-mon[27365]: pgmap v117: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-09-14T03:37:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:53 smithi110 ceph-mon[27365]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:37:53.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:53 smithi195 ceph-mon[30695]: pgmap v117: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-09-14T03:37:53.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:53 smithi195 ceph-mon[30695]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:37:54.092 INFO:tasks.rados.rados.0.smithi110.stdout:131: done (0 left) 2024-09-14T03:37:54.092 INFO:tasks.rados.rados.0.smithi110.stdout:132: rollback oid 17 current snap is 6 2024-09-14T03:37:54.092 INFO:tasks.rados.rados.0.smithi110.stdout:132: done (0 left) 2024-09-14T03:37:54.092 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 6 2024-09-14T03:37:54.093 INFO:tasks.rados.rados.0.smithi110.stdout:133: seq_num 65 ranges {712704=696320,1409024=401408} 2024-09-14T03:37:54.099 INFO:tasks.rados.rados.0.smithi110.stdout:133: writing smithi11075562-43 from 712704 to 1409024 tid 1 2024-09-14T03:37:54.103 INFO:tasks.rados.rados.0.smithi110.stdout:133: writing smithi11075562-43 from 1409024 to 1810432 tid 2 2024-09-14T03:37:54.103 INFO:tasks.rados.rados.0.smithi110.stdout:134: read oid 11 snap -1 2024-09-14T03:37:54.103 INFO:tasks.rados.rados.0.smithi110.stdout:134: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-14T03:37:54.103 INFO:tasks.rados.rados.0.smithi110.stdout:135: read oid 6 snap -1 2024-09-14T03:37:54.103 INFO:tasks.rados.rados.0.smithi110.stdout:135: expect deleted 2024-09-14T03:37:54.103 INFO:tasks.rados.rados.0.smithi110.stdout:136: copy_from oid 18 from oid 46 current snap is 6 2024-09-14T03:37:54.103 INFO:tasks.rados.rados.0.smithi110.stdout:137: read oid 41 snap -1 2024-09-14T03:37:54.103 INFO:tasks.rados.rados.0.smithi110.stdout:137: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-14T03:37:54.104 INFO:tasks.rados.rados.0.smithi110.stdout:138: rmattr oid 37 current snap is 6 2024-09-14T03:37:54.104 INFO:tasks.rados.rados.0.smithi110.stdout:138: done (5 left) 2024-09-14T03:37:54.104 INFO:tasks.rados.rados.0.smithi110.stdout:139: read oid 1 snap -1 2024-09-14T03:37:54.104 INFO:tasks.rados.rados.0.smithi110.stdout:139: expect deleted 2024-09-14T03:37:54.104 INFO:tasks.rados.rados.0.smithi110.stdout:140: rmattr oid 30 current snap is 6 2024-09-14T03:37:54.104 INFO:tasks.rados.rados.0.smithi110.stdout:140: done (6 left) 2024-09-14T03:37:54.104 INFO:tasks.rados.rados.0.smithi110.stdout:141: setattr oid 47 current snap is 6 2024-09-14T03:37:54.105 INFO:tasks.rados.rados.0.smithi110.stdout:135: done (6 left) 2024-09-14T03:37:54.105 INFO:tasks.rados.rados.0.smithi110.stdout:139: done (5 left) 2024-09-14T03:37:54.105 INFO:tasks.rados.rados.0.smithi110.stdout:142: copy_from oid 23 from oid 25 current snap is 6 2024-09-14T03:37:54.105 INFO:tasks.rados.rados.0.smithi110.stdout:143: setattr oid 24 current snap is 6 2024-09-14T03:37:54.105 INFO:tasks.rados.rados.0.smithi110.stdout:133: finishing write tid 1 to smithi11075562-43 2024-09-14T03:37:54.105 INFO:tasks.rados.rados.0.smithi110.stdout:133: finishing write tid 2 to smithi11075562-43 2024-09-14T03:37:54.105 INFO:tasks.rados.rados.0.smithi110.stdout:133: finishing write tid 3 to smithi11075562-43 2024-09-14T03:37:54.106 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 6 2024-09-14T03:37:54.106 INFO:tasks.rados.rados.0.smithi110.stdout:144: seq_num 66 ranges {303104=712704,1015808=401408} 2024-09-14T03:37:54.112 INFO:tasks.rados.rados.0.smithi110.stdout:144: writing smithi11075562-19 from 303104 to 1015808 tid 1 2024-09-14T03:37:54.115 INFO:tasks.rados.rados.0.smithi110.stdout:144: writing smithi11075562-19 from 1015808 to 1417216 tid 2 2024-09-14T03:37:54.115 INFO:tasks.rados.rados.0.smithi110.stdout:145: setattr oid 33 current snap is 6 2024-09-14T03:37:54.115 INFO:tasks.rados.rados.0.smithi110.stdout:133: finishing write tid 4 to smithi11075562-43 2024-09-14T03:37:54.115 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 35 (ObjNum 65 snap 6 seq_num 65) dirty exists 2024-09-14T03:37:54.115 INFO:tasks.rados.rados.0.smithi110.stdout:133: left oid 43 (ObjNum 65 snap 6 seq_num 65) 2024-09-14T03:37:54.116 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 51 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:37:54.116 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 37 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-14T03:37:54.116 INFO:tasks.rados.rados.0.smithi110.stdout:137: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-14T03:37:54.134 INFO:tasks.rados.rados.0.smithi110.stdout:134: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-14T03:37:54.139 INFO:tasks.rados.rados.0.smithi110.stdout:136: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:37:54.139 INFO:tasks.rados.rados.0.smithi110.stdout:142: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:37:54.139 INFO:tasks.rados.rados.0.smithi110.stdout:136: finishing copy_from to smithi11075562-18 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 22 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:142: finishing copy_from to smithi11075562-23 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 10 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:144: finishing write tid 1 to smithi11075562-19 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:144: finishing write tid 2 to smithi11075562-19 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:144: finishing write tid 3 to smithi11075562-19 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:144: finishing write tid 4 to smithi11075562-19 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 36 (ObjNum 66 snap 6 seq_num 66) dirty exists 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:144: left oid 19 (ObjNum 66 snap 6 seq_num 66) 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:133: done (8 left) 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:134: done (7 left) 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:136: done (6 left) 2024-09-14T03:37:54.140 INFO:tasks.rados.rados.0.smithi110.stdout:137: done (5 left) 2024-09-14T03:37:54.141 INFO:tasks.rados.rados.0.smithi110.stdout:141: done (4 left) 2024-09-14T03:37:54.141 INFO:tasks.rados.rados.0.smithi110.stdout:142: done (3 left) 2024-09-14T03:37:54.141 INFO:tasks.rados.rados.0.smithi110.stdout:143: done (2 left) 2024-09-14T03:37:54.141 INFO:tasks.rados.rados.0.smithi110.stdout:144: done (1 left) 2024-09-14T03:37:54.141 INFO:tasks.rados.rados.0.smithi110.stdout:146: snap_create 2024-09-14T03:37:54.141 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 35 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-14T03:37:54.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:54 smithi110 ceph-mon[35374]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:37:54.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:54 smithi110 ceph-mon[27365]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:37:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:54 smithi195 ceph-mon[30695]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:37:55.006 INFO:tasks.rados.rados.0.smithi110.stdout:145: done (1 left) 2024-09-14T03:37:55.007 INFO:tasks.rados.rados.0.smithi110.stdout:146: done (0 left) 2024-09-14T03:37:55.007 INFO:tasks.rados.rados.0.smithi110.stdout:147: snap_remove snap 6 2024-09-14T03:37:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:55 smithi110 ceph-mon[35374]: pgmap v120: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-09-14T03:37:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:55 smithi110 ceph-mon[35374]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:37:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:55 smithi110 ceph-mon[35374]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:37:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:55 smithi110 ceph-mon[27365]: pgmap v120: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-09-14T03:37:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:55 smithi110 ceph-mon[27365]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:37:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:55 smithi110 ceph-mon[27365]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:37:55.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:55 smithi195 ceph-mon[30695]: pgmap v120: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-09-14T03:37:55.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:55 smithi195 ceph-mon[30695]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:37:55.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:55 smithi195 ceph-mon[30695]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:37:56.008 INFO:tasks.rados.rados.0.smithi110.stdout:147: done (0 left) 2024-09-14T03:37:56.008 INFO:tasks.rados.rados.0.smithi110.stdout:148: delete oid 1 current snap is 7 2024-09-14T03:37:56.011 INFO:tasks.rados.rados.0.smithi110.stdout:148: done (0 left) 2024-09-14T03:37:56.011 INFO:tasks.rados.rados.0.smithi110.stdout:149: snap_create 2024-09-14T03:37:57.013 INFO:tasks.rados.rados.0.smithi110.stdout:149: done (0 left) 2024-09-14T03:37:57.013 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 8 2024-09-14T03:37:57.013 INFO:tasks.rados.rados.0.smithi110.stdout:150: seq_num 68 ranges {606208=745472,1351680=401408} 2024-09-14T03:37:57.019 INFO:tasks.rados.rados.0.smithi110.stdout:150: writing smithi11075562-31 from 606208 to 1351680 tid 1 2024-09-14T03:37:57.022 INFO:tasks.rados.rados.0.smithi110.stdout:150: writing smithi11075562-31 from 1351680 to 1753088 tid 2 2024-09-14T03:37:57.022 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 8 2024-09-14T03:37:57.022 INFO:tasks.rados.rados.0.smithi110.stdout:151: seq_num 69 ranges {253952=761856,1015808=401408} 2024-09-14T03:37:57.028 INFO:tasks.rados.rados.0.smithi110.stdout:151: writing smithi11075562-16 from 253952 to 1015808 tid 1 2024-09-14T03:37:57.031 INFO:tasks.rados.rados.0.smithi110.stdout:151: writing smithi11075562-16 from 1015808 to 1417216 tid 2 2024-09-14T03:37:57.031 INFO:tasks.rados.rados.0.smithi110.stdout:152: copy_from oid 10 from oid 39 current snap is 8 2024-09-14T03:37:57.031 INFO:tasks.rados.rados.0.smithi110.stdout:150: finishing write tid 1 to smithi11075562-31 2024-09-14T03:37:57.031 INFO:tasks.rados.rados.0.smithi110.stdout:150: finishing write tid 2 to smithi11075562-31 2024-09-14T03:37:57.031 INFO:tasks.rados.rados.0.smithi110.stdout:150: finishing write tid 3 to smithi11075562-31 2024-09-14T03:37:57.031 INFO:tasks.rados.rados.0.smithi110.stdout:150: finishing write tid 4 to smithi11075562-31 2024-09-14T03:37:57.031 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 22 (ObjNum 68 snap 8 seq_num 68) dirty exists 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:150: left oid 31 (ObjNum 68 snap 8 seq_num 68) 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:150: done (2 left) 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:153: read oid 25 snap 7 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:153: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:154: copy_from oid 21 from oid 35 current snap is 8 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:155: rollback oid 19 current snap is 8 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 7 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:156: copy_from oid 38 from oid 24 current snap is 8 2024-09-14T03:37:57.032 INFO:tasks.rados.rados.0.smithi110.stdout:157: snap_create 2024-09-14T03:37:57.034 INFO:tasks.rados.rados.0.smithi110.stdout:151: finishing write tid 1 to smithi11075562-16 2024-09-14T03:37:57.034 INFO:tasks.rados.rados.0.smithi110.stdout:151: finishing write tid 2 to smithi11075562-16 2024-09-14T03:37:57.036 INFO:tasks.rados.rados.0.smithi110.stdout:153: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-14T03:37:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:151: finishing write tid 3 to smithi11075562-16 2024-09-14T03:37:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:155: finishing rollback tid 0 to smithi11075562-19 2024-09-14T03:37:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:155: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:37:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:151: finishing write tid 4 to smithi11075562-16 2024-09-14T03:37:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 40 (ObjNum 69 snap 8 seq_num 69) dirty exists 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:151: left oid 16 (ObjNum 69 snap 8 seq_num 69) 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:155: finishing rollback tid 2 to smithi11075562-19 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 44 (ObjNum 66 snap 6 seq_num 66) dirty exists 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:152: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:152: finishing copy_from to smithi11075562-10 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:156: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:154: finishing copy_from racing read to smithi11075562-21 2024-09-14T03:37:57.057 INFO:tasks.rados.rados.0.smithi110.stdout:156: finishing copy_from to smithi11075562-38 2024-09-14T03:37:57.058 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 37 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-14T03:37:57.058 INFO:tasks.rados.rados.0.smithi110.stdout:154: finishing copy_from to smithi11075562-21 2024-09-14T03:37:57.058 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 20 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-14T03:37:57.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:57 smithi110 ceph-mon[35374]: pgmap v123: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:57.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:57 smithi110 ceph-mon[35374]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:37:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:57 smithi110 ceph-mon[27365]: pgmap v123: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:57 smithi110 ceph-mon[27365]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:37:57.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:57 smithi195 ceph-mon[30695]: pgmap v123: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:37:57.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:57 smithi195 ceph-mon[30695]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:37:58.019 INFO:tasks.rados.rados.0.smithi110.stdout:151: done (6 left) 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:152: done (5 left) 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:153: done (4 left) 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:154: done (3 left) 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:155: done (2 left) 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:156: done (1 left) 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:157: done (0 left) 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:158: read oid 46 snap -1 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:158: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:37:58.020 INFO:tasks.rados.rados.0.smithi110.stdout:159: copy_from oid 3 from oid 30 current snap is 9 2024-09-14T03:37:58.021 INFO:tasks.rados.rados.0.smithi110.stdout:160: copy_from oid 17 from oid 9 current snap is 9 2024-09-14T03:37:58.021 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 9 2024-09-14T03:37:58.021 INFO:tasks.rados.rados.0.smithi110.stdout:161: seq_num 70 ranges {1753088=778240,2531328=401408} 2024-09-14T03:37:58.034 INFO:tasks.rados.rados.0.smithi110.stdout:161: writing smithi11075562-31 from 1753088 to 2531328 tid 1 2024-09-14T03:37:58.037 INFO:tasks.rados.rados.0.smithi110.stdout:161: writing smithi11075562-31 from 2531328 to 2932736 tid 2 2024-09-14T03:37:58.037 INFO:tasks.rados.rados.0.smithi110.stdout:162: setattr oid 32 current snap is 9 2024-09-14T03:37:58.037 INFO:tasks.rados.rados.0.smithi110.stdout:158: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:37:58.061 INFO:tasks.rados.rados.0.smithi110.stdout:160: finishing copy_from racing read to smithi11075562-17 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:159: finishing copy_from racing read to smithi11075562-3 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:160: finishing copy_from to smithi11075562-17 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 14 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:159: finishing copy_from to smithi11075562-3 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 8 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:161: finishing write tid 1 to smithi11075562-31 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:161: finishing write tid 2 to smithi11075562-31 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:161: finishing write tid 3 to smithi11075562-31 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:161: finishing write tid 4 to smithi11075562-31 2024-09-14T03:37:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 26 (ObjNum 70 snap 9 seq_num 70) dirty exists 2024-09-14T03:37:58.063 INFO:tasks.rados.rados.0.smithi110.stdout:161: left oid 31 (ObjNum 70 snap 9 seq_num 70) 2024-09-14T03:37:58.063 INFO:tasks.rados.rados.0.smithi110.stdout:158: done (4 left) 2024-09-14T03:37:58.063 INFO:tasks.rados.rados.0.smithi110.stdout:159: done (3 left) 2024-09-14T03:37:58.063 INFO:tasks.rados.rados.0.smithi110.stdout:160: done (2 left) 2024-09-14T03:37:58.063 INFO:tasks.rados.rados.0.smithi110.stdout:161: done (1 left) 2024-09-14T03:37:58.063 INFO:tasks.rados.rados.0.smithi110.stdout:163: snap_remove snap 8 2024-09-14T03:37:58.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:58 smithi110 ceph-mon[35374]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:37:58.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:58 smithi110 ceph-mon[27365]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:37:58.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:58 smithi195 ceph-mon[30695]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:37:59.023 INFO:tasks.rados.rados.0.smithi110.stdout:163: done (1 left) 2024-09-14T03:37:59.023 INFO:tasks.rados.rados.0.smithi110.stdout:164: rmattr oid 36 current snap is 9 2024-09-14T03:37:59.023 INFO:tasks.rados.rados.0.smithi110.stdout:164: done (1 left) 2024-09-14T03:37:59.023 INFO:tasks.rados.rados.0.smithi110.stdout:165: copy_from oid 6 from oid 13 current snap is 9 2024-09-14T03:37:59.023 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 33 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-14T03:37:59.023 INFO:tasks.rados.rados.0.smithi110.stdout:162: done (1 left) 2024-09-14T03:37:59.023 INFO:tasks.rados.rados.0.smithi110.stdout:166: snap_remove snap 7 2024-09-14T03:37:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:59 smithi110 ceph-mon[27365]: pgmap v126: 121 pgs: 121 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 874 KiB/s wr, 34 op/s 2024-09-14T03:37:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:37:59 smithi110 ceph-mon[27365]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:37:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:59 smithi110 ceph-mon[35374]: pgmap v126: 121 pgs: 121 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 874 KiB/s wr, 34 op/s 2024-09-14T03:37:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:37:59 smithi110 ceph-mon[35374]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:37:59.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:59 smithi195 ceph-mon[30695]: pgmap v126: 121 pgs: 121 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 874 KiB/s wr, 34 op/s 2024-09-14T03:37:59.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:37:59 smithi195 ceph-mon[30695]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:38:00.008 INFO:tasks.rados.rados.0.smithi110.stdout:166: done (1 left) 2024-09-14T03:38:00.008 INFO:tasks.rados.rados.0.smithi110.stdout:167: read oid 29 snap -1 2024-09-14T03:38:00.008 INFO:tasks.rados.rados.0.smithi110.stdout:167: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:38:00.009 INFO:tasks.rados.rados.0.smithi110.stdout:168: rollback oid 35 current snap is 9 2024-09-14T03:38:00.009 INFO:tasks.rados.rados.0.smithi110.stdout:168: done (2 left) 2024-09-14T03:38:00.009 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 9 2024-09-14T03:38:00.009 INFO:tasks.rados.rados.0.smithi110.stdout:169: seq_num 71 ranges {24576=794624,819200=401408} 2024-09-14T03:38:00.014 INFO:tasks.rados.rados.0.smithi110.stdout:169: writing smithi11075562-2 from 24576 to 819200 tid 1 2024-09-14T03:38:00.022 INFO:tasks.rados.rados.0.smithi110.stdout:169: writing smithi11075562-2 from 819200 to 1220608 tid 2 2024-09-14T03:38:00.022 INFO:tasks.rados.rados.0.smithi110.stdout:165: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:38:00.022 INFO:tasks.rados.rados.0.smithi110.stdout:165: finishing copy_from to smithi11075562-6 2024-09-14T03:38:00.022 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 4 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-14T03:38:00.022 INFO:tasks.rados.rados.0.smithi110.stdout:167: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:38:00.047 INFO:tasks.rados.rados.0.smithi110.stdout:165: done (2 left) 2024-09-14T03:38:00.048 INFO:tasks.rados.rados.0.smithi110.stdout:167: done (1 left) 2024-09-14T03:38:00.048 INFO:tasks.rados.rados.0.smithi110.stdout:170: read oid 35 snap -1 2024-09-14T03:38:00.048 INFO:tasks.rados.rados.0.smithi110.stdout:170: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-14T03:38:00.048 INFO:tasks.rados.rados.0.smithi110.stdout:171: snap_create 2024-09-14T03:38:00.048 INFO:tasks.rados.rados.0.smithi110.stdout:169: finishing write tid 1 to smithi11075562-2 2024-09-14T03:38:00.048 INFO:tasks.rados.rados.0.smithi110.stdout:169: finishing write tid 2 to smithi11075562-2 2024-09-14T03:38:00.049 INFO:tasks.rados.rados.0.smithi110.stdout:169: finishing write tid 3 to smithi11075562-2 2024-09-14T03:38:00.049 INFO:tasks.rados.rados.0.smithi110.stdout:169: finishing write tid 4 to smithi11075562-2 2024-09-14T03:38:00.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 21 (ObjNum 71 snap 9 seq_num 71) dirty exists 2024-09-14T03:38:00.049 INFO:tasks.rados.rados.0.smithi110.stdout:169: left oid 2 (ObjNum 71 snap 9 seq_num 71) 2024-09-14T03:38:00.051 INFO:tasks.rados.rados.0.smithi110.stdout:170: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-14T03:38:00.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:00 smithi110 ceph-mon[35374]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:38:00.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:00 smithi110 ceph-mon[35374]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:38:00.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:00 smithi110 ceph-mon[27365]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:38:00.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:00 smithi110 ceph-mon[27365]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:38:00.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:38:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:00 smithi195 ceph-mon[30695]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:38:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:00 smithi195 ceph-mon[30695]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:38:00.665 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:00] "GET /metrics HTTP/1.1" 200 239649 "" "Prometheus/2.43.0" 2024-09-14T03:38:01.031 INFO:tasks.rados.rados.0.smithi110.stdout:169: done (2 left) 2024-09-14T03:38:01.031 INFO:tasks.rados.rados.0.smithi110.stdout:170: done (1 left) 2024-09-14T03:38:01.031 INFO:tasks.rados.rados.0.smithi110.stdout:171: done (0 left) 2024-09-14T03:38:01.031 INFO:tasks.rados.rados.0.smithi110.stdout:172: delete oid 44 current snap is 10 2024-09-14T03:38:01.036 INFO:tasks.rados.rados.0.smithi110.stdout:172: done (0 left) 2024-09-14T03:38:01.036 INFO:tasks.rados.rados.0.smithi110.stdout:173: snap_create 2024-09-14T03:38:01.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:01 smithi110 ceph-mon[35374]: pgmap v129: 121 pgs: 121 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 874 KiB/s wr, 34 op/s 2024-09-14T03:38:01.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:01 smithi110 ceph-mon[27365]: pgmap v129: 121 pgs: 121 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 874 KiB/s wr, 34 op/s 2024-09-14T03:38:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:01 smithi195 ceph-mon[30695]: pgmap v129: 121 pgs: 121 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 874 KiB/s wr, 34 op/s 2024-09-14T03:38:02.040 INFO:tasks.rados.rados.0.smithi110.stdout:173: done (0 left) 2024-09-14T03:38:02.040 INFO:tasks.rados.rados.0.smithi110.stdout:174: read oid 40 snap -1 2024-09-14T03:38:02.040 INFO:tasks.rados.rados.0.smithi110.stdout:174: expect (ObjNum 61 snap 5 seq_num 61) 2024-09-14T03:38:02.040 INFO:tasks.rados.rados.0.smithi110.stdout:175: snap_create 2024-09-14T03:38:02.050 INFO:tasks.rados.rados.0.smithi110.stdout:174: expect (ObjNum 61 snap 5 seq_num 61) 2024-09-14T03:38:02.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:02 smithi110 ceph-mon[35374]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:38:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:02 smithi110 ceph-mon[27365]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:38:02.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:02 smithi195 ceph-mon[30695]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:38:03.044 INFO:tasks.rados.rados.0.smithi110.stdout:174: done (1 left) 2024-09-14T03:38:03.044 INFO:tasks.rados.rados.0.smithi110.stdout:175: done (0 left) 2024-09-14T03:38:03.044 INFO:tasks.rados.rados.0.smithi110.stdout:176: snap_remove snap 9 2024-09-14T03:38:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:03 smithi110 ceph-mon[35374]: pgmap v132: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-14T03:38:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:03 smithi110 ceph-mon[35374]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:38:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:03 smithi110 ceph-mon[27365]: pgmap v132: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-14T03:38:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:03 smithi110 ceph-mon[27365]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:38:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:03 smithi195 ceph-mon[30695]: pgmap v132: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-14T03:38:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:03 smithi195 ceph-mon[30695]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:38:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:176: done (0 left) 2024-09-14T03:38:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:177: read oid 36 snap -1 2024-09-14T03:38:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:177: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:38:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 12 2024-09-14T03:38:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:178: seq_num 73 ranges {0=425984,425984=409600,835584=393216} 2024-09-14T03:38:04.048 INFO:tasks.rados.rados.0.smithi110.stdout:178: writing smithi11075562-1 from 0 to 425984 tid 1 2024-09-14T03:38:04.050 INFO:tasks.rados.rados.0.smithi110.stdout:178: writing smithi11075562-1 from 425984 to 835584 tid 2 2024-09-14T03:38:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:178: writing smithi11075562-1 from 835584 to 1228800 tid 3 2024-09-14T03:38:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 12 2024-09-14T03:38:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:179: seq_num 74 ranges {139264=442368,581632=696320,1277952=106496} 2024-09-14T03:38:04.056 INFO:tasks.rados.rados.0.smithi110.stdout:179: writing smithi11075562-9 from 139264 to 581632 tid 1 2024-09-14T03:38:04.060 INFO:tasks.rados.rados.0.smithi110.stdout:179: writing smithi11075562-9 from 581632 to 1277952 tid 2 2024-09-14T03:38:04.061 INFO:tasks.rados.rados.0.smithi110.stdout:179: writing smithi11075562-9 from 1277952 to 1384448 tid 3 2024-09-14T03:38:04.061 INFO:tasks.rados.rados.0.smithi110.stdout:178: finishing write tid 1 to smithi11075562-1 2024-09-14T03:38:04.061 INFO:tasks.rados.rados.0.smithi110.stdout:178: finishing write tid 2 to smithi11075562-1 2024-09-14T03:38:04.061 INFO:tasks.rados.rados.0.smithi110.stdout:177: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:178: finishing write tid 3 to smithi11075562-1 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:178: finishing write tid 4 to smithi11075562-1 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:178: finishing write tid 5 to smithi11075562-1 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 31 (ObjNum 73 snap 12 seq_num 73) dirty exists 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:178: left oid 1 (ObjNum 73 snap 12 seq_num 73) 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:177: done (2 left) 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:178: done (1 left) 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:180: rmattr oid 46 current snap is 12 2024-09-14T03:38:04.062 INFO:tasks.rados.rados.0.smithi110.stdout:181: snap_remove snap 11 2024-09-14T03:38:04.178 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:04 smithi110 ceph-mon[35374]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:38:04.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:04 smithi110 ceph-mon[27365]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:38:04.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:04 smithi195 ceph-mon[30695]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:38:05.009 INFO:tasks.rados.rados.0.smithi110.stdout:179: finishing write tid 1 to smithi11075562-9 2024-09-14T03:38:05.009 INFO:tasks.rados.rados.0.smithi110.stdout:179: finishing write tid 2 to smithi11075562-9 2024-09-14T03:38:05.009 INFO:tasks.rados.rados.0.smithi110.stdout:179: finishing write tid 3 to smithi11075562-9 2024-09-14T03:38:05.009 INFO:tasks.rados.rados.0.smithi110.stdout:179: finishing write tid 4 to smithi11075562-9 2024-09-14T03:38:05.009 INFO:tasks.rados.rados.0.smithi110.stdout:179: finishing write tid 5 to smithi11075562-9 2024-09-14T03:38:05.009 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 40 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-14T03:38:05.009 INFO:tasks.rados.rados.0.smithi110.stdout:179: left oid 9 (ObjNum 74 snap 12 seq_num 74) 2024-09-14T03:38:05.009 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 42 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:38:05.010 INFO:tasks.rados.rados.0.smithi110.stdout:179: done (2 left) 2024-09-14T03:38:05.010 INFO:tasks.rados.rados.0.smithi110.stdout:180: done (1 left) 2024-09-14T03:38:05.010 INFO:tasks.rados.rados.0.smithi110.stdout:181: done (0 left) 2024-09-14T03:38:05.010 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 12 2024-09-14T03:38:05.010 INFO:tasks.rados.rados.0.smithi110.stdout:182: seq_num 75 ranges {0=458752,458752=581632,1040384=221184} 2024-09-14T03:38:05.011 INFO:tasks.rados.rados.0.smithi110.stdout:182: writing smithi11075562-45 from 0 to 458752 tid 1 2024-09-14T03:38:05.014 INFO:tasks.rados.rados.0.smithi110.stdout:182: writing smithi11075562-45 from 458752 to 1040384 tid 2 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:182: writing smithi11075562-45 from 1040384 to 1261568 tid 3 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:183: copy_from oid 28 from oid 4 current snap is 12 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:184: rollback oid 37 current snap is 12 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 37 to 10 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:185: rmattr oid 8 current snap is 12 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:185: done (3 left) 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:186: rmattr oid 23 current snap is 12 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:186: done (3 left) 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:187: rmattr oid 39 current snap is 12 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:187: done (3 left) 2024-09-14T03:38:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:188: read oid 1 snap -1 2024-09-14T03:38:05.018 INFO:tasks.rados.rados.0.smithi110.stdout:188: expect (ObjNum 73 snap 12 seq_num 73) 2024-09-14T03:38:05.018 INFO:tasks.rados.rados.0.smithi110.stdout:189: read oid 6 snap -1 2024-09-14T03:38:05.018 INFO:tasks.rados.rados.0.smithi110.stdout:189: expect (ObjNum 12 snap 0 seq_num 12) 2024-09-14T03:38:05.018 INFO:tasks.rados.rados.0.smithi110.stdout:190: setattr oid 35 current snap is 12 2024-09-14T03:38:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 12 2024-09-14T03:38:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:191: seq_num 76 ranges {1220608=475136,1695744=458752,2154496=344064} 2024-09-14T03:38:05.029 INFO:tasks.rados.rados.0.smithi110.stdout:191: writing smithi11075562-2 from 1220608 to 1695744 tid 1 2024-09-14T03:38:05.032 INFO:tasks.rados.rados.0.smithi110.stdout:191: writing smithi11075562-2 from 1695744 to 2154496 tid 2 2024-09-14T03:38:05.034 INFO:tasks.rados.rados.0.smithi110.stdout:191: writing smithi11075562-2 from 2154496 to 2498560 tid 3 2024-09-14T03:38:05.034 INFO:tasks.rados.rados.0.smithi110.stdout:192: copy_from oid 33 from oid 22 current snap is 12 2024-09-14T03:38:05.034 INFO:tasks.rados.rados.0.smithi110.stdout:189: expect (ObjNum 12 snap 0 seq_num 12) 2024-09-14T03:38:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:183: finishing copy_from racing read to smithi11075562-28 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:182: finishing write tid 1 to smithi11075562-45 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:183: finishing copy_from to smithi11075562-28 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:182: finishing write tid 2 to smithi11075562-45 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:182: finishing write tid 3 to smithi11075562-45 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:182: finishing write tid 4 to smithi11075562-45 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:184: finishing rollback tid 0 to smithi11075562-37 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:184: finishing rollback tid 1 to smithi11075562-37 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:184: finishing rollback tid 2 to smithi11075562-37 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 59 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:182: finishing write tid 5 to smithi11075562-45 2024-09-14T03:38:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 55 (ObjNum 75 snap 12 seq_num 75) dirty exists 2024-09-14T03:38:05.042 INFO:tasks.rados.rados.0.smithi110.stdout:182: left oid 45 (ObjNum 75 snap 12 seq_num 75) 2024-09-14T03:38:05.042 INFO:tasks.rados.rados.0.smithi110.stdout:191: finishing write tid 1 to smithi11075562-2 2024-09-14T03:38:05.042 INFO:tasks.rados.rados.0.smithi110.stdout:188: expect (ObjNum 73 snap 12 seq_num 73) 2024-09-14T03:38:05.073 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 33 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-14T03:38:05.073 INFO:tasks.rados.rados.0.smithi110.stdout:191: finishing write tid 2 to smithi11075562-2 2024-09-14T03:38:05.073 INFO:tasks.rados.rados.0.smithi110.stdout:191: finishing write tid 3 to smithi11075562-2 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:182: done (7 left) 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:183: done (6 left) 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:184: done (5 left) 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:188: done (4 left) 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:189: done (3 left) 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:190: done (2 left) 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:193: snap_create 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:191: finishing write tid 4 to smithi11075562-2 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:191: finishing write tid 5 to smithi11075562-2 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 28 (ObjNum 76 snap 12 seq_num 76) dirty exists 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:191: left oid 2 (ObjNum 76 snap 12 seq_num 76) 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:192: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:192: finishing copy_from to smithi11075562-33 2024-09-14T03:38:05.074 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 6 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-14T03:38:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[27365]: pgmap v135: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-14T03:38:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[27365]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:38:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[27365]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:38:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[35374]: pgmap v135: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-14T03:38:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[35374]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:38:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:05 smithi110 ceph-mon[35374]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:38:05.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:05 smithi195 ceph-mon[30695]: pgmap v135: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-14T03:38:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:05 smithi195 ceph-mon[30695]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:38:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:05 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:05 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:05 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:05 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:05 smithi195 ceph-mon[30695]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:38:06.057 INFO:tasks.rados.rados.0.smithi110.stdout:191: done (2 left) 2024-09-14T03:38:06.057 INFO:tasks.rados.rados.0.smithi110.stdout:192: done (1 left) 2024-09-14T03:38:06.057 INFO:tasks.rados.rados.0.smithi110.stdout:193: done (0 left) 2024-09-14T03:38:06.057 INFO:tasks.rados.rados.0.smithi110.stdout:194: delete oid 31 current snap is 13 2024-09-14T03:38:06.058 INFO:tasks.rados.rados.0.smithi110.stdout:194: done (0 left) 2024-09-14T03:38:06.058 INFO:tasks.rados.rados.0.smithi110.stdout:195: delete oid 26 current snap is 13 2024-09-14T03:38:06.061 INFO:tasks.rados.rados.0.smithi110.stdout:195: done (0 left) 2024-09-14T03:38:06.061 INFO:tasks.rados.rados.0.smithi110.stdout:196: copy_from oid 20 from oid 42 current snap is 13 2024-09-14T03:38:06.061 INFO:tasks.rados.rados.0.smithi110.stdout:197: delete oid 5 current snap is 13 2024-09-14T03:38:06.064 INFO:tasks.rados.rados.0.smithi110.stdout:197: done (1 left) 2024-09-14T03:38:06.064 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 13 2024-09-14T03:38:06.064 INFO:tasks.rados.rados.0.smithi110.stdout:198: seq_num 80 ranges {204800=548864,753664=450560,1204224=352256} 2024-09-14T03:38:06.068 INFO:tasks.rados.rados.0.smithi110.stdout:198: writing smithi11075562-13 from 204800 to 753664 tid 1 2024-09-14T03:38:06.070 INFO:tasks.rados.rados.0.smithi110.stdout:198: writing smithi11075562-13 from 753664 to 1204224 tid 2 2024-09-14T03:38:06.073 INFO:tasks.rados.rados.0.smithi110.stdout:198: writing smithi11075562-13 from 1204224 to 1556480 tid 3 2024-09-14T03:38:06.073 INFO:tasks.rados.rados.0.smithi110.stdout:199: read oid 19 snap -1 2024-09-14T03:38:06.074 INFO:tasks.rados.rados.0.smithi110.stdout:199: expect (ObjNum 66 snap 6 seq_num 66) 2024-09-14T03:38:06.074 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 13 2024-09-14T03:38:06.074 INFO:tasks.rados.rados.0.smithi110.stdout:196: finishing copy_from racing read to smithi11075562-20 2024-09-14T03:38:06.074 INFO:tasks.rados.rados.0.smithi110.stdout:196: finishing copy_from to smithi11075562-20 2024-09-14T03:38:06.074 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 8 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-14T03:38:06.074 INFO:tasks.rados.rados.0.smithi110.stdout:198: finishing write tid 1 to smithi11075562-13 2024-09-14T03:38:06.074 INFO:tasks.rados.rados.0.smithi110.stdout:200: seq_num 81 ranges {573440=565248,1138688=737280,1875968=65536} 2024-09-14T03:38:06.079 INFO:tasks.rados.rados.0.smithi110.stdout:200: writing smithi11075562-21 from 573440 to 1138688 tid 1 2024-09-14T03:38:06.084 INFO:tasks.rados.rados.0.smithi110.stdout:200: writing smithi11075562-21 from 1138688 to 1875968 tid 2 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:200: writing smithi11075562-21 from 1875968 to 1941504 tid 3 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:196: done (3 left) 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:201: delete oid 11 current snap is 13 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:198: finishing write tid 2 to smithi11075562-13 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:198: finishing write tid 3 to smithi11075562-13 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:198: finishing write tid 4 to smithi11075562-13 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:198: finishing write tid 5 to smithi11075562-13 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 38 (ObjNum 80 snap 13 seq_num 80) dirty exists 2024-09-14T03:38:06.085 INFO:tasks.rados.rados.0.smithi110.stdout:198: left oid 13 (ObjNum 80 snap 13 seq_num 80) 2024-09-14T03:38:06.086 INFO:tasks.rados.rados.0.smithi110.stdout:199: expect (ObjNum 66 snap 6 seq_num 66) 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:200: finishing write tid 1 to smithi11075562-21 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:200: finishing write tid 2 to smithi11075562-21 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:200: finishing write tid 3 to smithi11075562-21 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:200: finishing write tid 4 to smithi11075562-21 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:200: finishing write tid 5 to smithi11075562-21 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 44 (ObjNum 81 snap 13 seq_num 81) dirty exists 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:200: left oid 21 (ObjNum 81 snap 13 seq_num 81) 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:198: done (3 left) 2024-09-14T03:38:06.128 INFO:tasks.rados.rados.0.smithi110.stdout:199: done (2 left) 2024-09-14T03:38:06.129 INFO:tasks.rados.rados.0.smithi110.stdout:200: done (1 left) 2024-09-14T03:38:06.129 INFO:tasks.rados.rados.0.smithi110.stdout:201: done (0 left) 2024-09-14T03:38:06.129 INFO:tasks.rados.rados.0.smithi110.stdout:202: read oid 38 snap -1 2024-09-14T03:38:06.129 INFO:tasks.rados.rados.0.smithi110.stdout:202: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:38:06.129 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 13 2024-09-14T03:38:06.129 INFO:tasks.rados.rados.0.smithi110.stdout:203: seq_num 83 ranges {2498560=598016,3096576=499712,3596288=303104} 2024-09-14T03:38:06.144 INFO:tasks.rados.rados.0.smithi110.stdout:203: writing smithi11075562-2 from 2498560 to 3096576 tid 1 2024-09-14T03:38:06.147 INFO:tasks.rados.rados.0.smithi110.stdout:203: writing smithi11075562-2 from 3096576 to 3596288 tid 2 2024-09-14T03:38:06.149 INFO:tasks.rados.rados.0.smithi110.stdout:203: writing smithi11075562-2 from 3596288 to 3899392 tid 3 2024-09-14T03:38:06.149 INFO:tasks.rados.rados.0.smithi110.stdout:204: delete oid 41 current snap is 13 2024-09-14T03:38:06.149 INFO:tasks.rados.rados.0.smithi110.stdout:202: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:38:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:203: finishing write tid 1 to smithi11075562-2 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:203: finishing write tid 2 to smithi11075562-2 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:203: finishing write tid 3 to smithi11075562-2 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:203: finishing write tid 4 to smithi11075562-2 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:203: finishing write tid 5 to smithi11075562-2 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 33 (ObjNum 83 snap 13 seq_num 83) dirty exists 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:203: left oid 2 (ObjNum 83 snap 13 seq_num 83) 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:202: done (2 left) 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:203: done (1 left) 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:204: done (0 left) 2024-09-14T03:38:06.192 INFO:tasks.rados.rados.0.smithi110.stdout:205: delete oid 35 current snap is 13 2024-09-14T03:38:06.193 INFO:tasks.rados.rados.0.smithi110.stdout:205: done (0 left) 2024-09-14T03:38:06.194 INFO:tasks.rados.rados.0.smithi110.stdout:206: setattr oid 29 current snap is 13 2024-09-14T03:38:06.195 INFO:tasks.rados.rados.0.smithi110.stdout:207: snap_remove snap 10 2024-09-14T03:38:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:06 smithi110 ceph-mon[27365]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:38:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:06 smithi110 ceph-mon[27365]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:38:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:06 smithi110 ceph-mon[27365]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:38:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:06 smithi110 ceph-mon[27365]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi195 2024-09-14T03:38:06.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:06 smithi110 ceph-mon[35374]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:38:06.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:06 smithi110 ceph-mon[35374]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:38:06.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:06 smithi110 ceph-mon[35374]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:38:06.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:06 smithi110 ceph-mon[35374]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi195 2024-09-14T03:38:06.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:06 smithi195 ceph-mon[30695]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:38:06.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:06 smithi195 ceph-mon[30695]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:38:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:06 smithi195 ceph-mon[30695]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:38:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:06 smithi195 ceph-mon[30695]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi195 2024-09-14T03:38:07.059 INFO:tasks.rados.rados.0.smithi110.stdout:207: done (1 left) 2024-09-14T03:38:07.059 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 13 2024-09-14T03:38:07.059 INFO:tasks.rados.rados.0.smithi110.stdout:208: seq_num 86 ranges {1302528=647168,1949696=606208,2555904=196608} 2024-09-14T03:38:07.069 INFO:tasks.rados.rados.0.smithi110.stdout:208: writing smithi11075562-38 from 1302528 to 1949696 tid 1 2024-09-14T03:38:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:208: writing smithi11075562-38 from 1949696 to 2555904 tid 2 2024-09-14T03:38:07.074 INFO:tasks.rados.rados.0.smithi110.stdout:208: writing smithi11075562-38 from 2555904 to 2752512 tid 3 2024-09-14T03:38:07.074 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 13 2024-09-14T03:38:07.074 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 38 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:38:07.074 INFO:tasks.rados.rados.0.smithi110.stdout:208: finishing write tid 1 to smithi11075562-38 2024-09-14T03:38:07.075 INFO:tasks.rados.rados.0.smithi110.stdout:209: seq_num 87 ranges {811008=663552,1474560=491520,1966080=311296} 2024-09-14T03:38:07.082 INFO:tasks.rados.rados.0.smithi110.stdout:209: writing smithi11075562-49 from 811008 to 1474560 tid 1 2024-09-14T03:38:07.085 INFO:tasks.rados.rados.0.smithi110.stdout:209: writing smithi11075562-49 from 1474560 to 1966080 tid 2 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:209: writing smithi11075562-49 from 1966080 to 2277376 tid 3 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:206: done (2 left) 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:210: copy_from oid 26 from oid 25 current snap is 13 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:208: finishing write tid 2 to smithi11075562-38 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:208: finishing write tid 3 to smithi11075562-38 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:208: finishing write tid 4 to smithi11075562-38 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:208: finishing write tid 5 to smithi11075562-38 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 42 (ObjNum 86 snap 13 seq_num 86) dirty exists 2024-09-14T03:38:07.088 INFO:tasks.rados.rados.0.smithi110.stdout:208: left oid 38 (ObjNum 86 snap 13 seq_num 86) 2024-09-14T03:38:07.089 INFO:tasks.rados.rados.0.smithi110.stdout:209: finishing write tid 1 to smithi11075562-49 2024-09-14T03:38:07.089 INFO:tasks.rados.rados.0.smithi110.stdout:208: done (2 left) 2024-09-14T03:38:07.089 INFO:tasks.rados.rados.0.smithi110.stdout:211: snap_create 2024-09-14T03:38:07.089 INFO:tasks.rados.rados.0.smithi110.stdout:209: finishing write tid 2 to smithi11075562-49 2024-09-14T03:38:07.089 INFO:tasks.rados.rados.0.smithi110.stdout:209: finishing write tid 3 to smithi11075562-49 2024-09-14T03:38:07.090 INFO:tasks.rados.rados.0.smithi110.stdout:209: finishing write tid 4 to smithi11075562-49 2024-09-14T03:38:07.091 INFO:tasks.rados.rados.0.smithi110.stdout:209: finishing write tid 5 to smithi11075562-49 2024-09-14T03:38:07.091 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 49 (ObjNum 87 snap 13 seq_num 87) dirty exists 2024-09-14T03:38:07.091 INFO:tasks.rados.rados.0.smithi110.stdout:209: left oid 49 (ObjNum 87 snap 13 seq_num 87) 2024-09-14T03:38:07.091 INFO:tasks.rados.rados.0.smithi110.stdout:210: finishing copy_from racing read to smithi11075562-26 2024-09-14T03:38:07.092 INFO:tasks.rados.rados.0.smithi110.stdout:210: finishing copy_from to smithi11075562-26 2024-09-14T03:38:07.092 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 10 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:38:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:07 smithi110 ceph-mon[35374]: pgmap v138: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:07 smithi110 ceph-mon[35374]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:38:07.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:07 smithi110 ceph-mon[27365]: pgmap v138: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:07.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:07 smithi110 ceph-mon[27365]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:38:07.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:07 smithi195 ceph-mon[30695]: pgmap v138: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:07 smithi195 ceph-mon[30695]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:38:08.069 INFO:tasks.rados.rados.0.smithi110.stdout:209: done (2 left) 2024-09-14T03:38:08.069 INFO:tasks.rados.rados.0.smithi110.stdout:210: done (1 left) 2024-09-14T03:38:08.069 INFO:tasks.rados.rados.0.smithi110.stdout:211: done (0 left) 2024-09-14T03:38:08.069 INFO:tasks.rados.rados.0.smithi110.stdout:212: snap_create 2024-09-14T03:38:08.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:08 smithi110 ceph-mon[35374]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:38:08.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:08 smithi110 ceph-mon[27365]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:38:08.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:08 smithi195 ceph-mon[30695]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:38:08.926 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:38:09.069 INFO:tasks.rados.rados.0.smithi110.stdout:212: done (0 left) 2024-09-14T03:38:09.069 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 15 2024-09-14T03:38:09.069 INFO:tasks.rados.rados.0.smithi110.stdout:213: seq_num 88 ranges {139264=679936,819200=778240,1597440=24576} 2024-09-14T03:38:09.073 INFO:tasks.rados.rados.0.smithi110.stdout:213: writing smithi11075562-17 from 139264 to 819200 tid 1 2024-09-14T03:38:09.078 INFO:tasks.rados.rados.0.smithi110.stdout:213: writing smithi11075562-17 from 819200 to 1597440 tid 2 2024-09-14T03:38:09.078 INFO:tasks.rados.rados.0.smithi110.stdout:213: writing smithi11075562-17 from 1597440 to 1622016 tid 3 2024-09-14T03:38:09.078 INFO:tasks.rados.rados.0.smithi110.stdout:214: rollback oid 8 current snap is 15 2024-09-14T03:38:09.078 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 8 to 13 2024-09-14T03:38:09.078 INFO:tasks.rados.rados.0.smithi110.stdout:213: finishing write tid 1 to smithi11075562-17 2024-09-14T03:38:09.079 INFO:tasks.rados.rados.0.smithi110.stdout:215: rmattr oid 26 current snap is 15 2024-09-14T03:38:09.079 INFO:tasks.rados.rados.0.smithi110.stdout:215: done (2 left) 2024-09-14T03:38:09.079 INFO:tasks.rados.rados.0.smithi110.stdout:216: delete oid 2 current snap is 15 2024-09-14T03:38:09.080 INFO:tasks.rados.rados.0.smithi110.stdout:214: finishing rollback tid 1 to smithi11075562-8 2024-09-14T03:38:09.080 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 0 (ObjNum 129 snap 0 seq_num 64918168) dirty dne 2024-09-14T03:38:09.081 INFO:tasks.rados.rados.0.smithi110.stdout:214: done (2 left) 2024-09-14T03:38:09.081 INFO:tasks.rados.rados.0.smithi110.stdout:216: done (1 left) 2024-09-14T03:38:09.081 INFO:tasks.rados.rados.0.smithi110.stdout:217: rmattr oid 10 current snap is 15 2024-09-14T03:38:09.081 INFO:tasks.rados.rados.0.smithi110.stdout:217: done (1 left) 2024-09-14T03:38:09.081 INFO:tasks.rados.rados.0.smithi110.stdout:218: snap_create 2024-09-14T03:38:09.082 INFO:tasks.rados.rados.0.smithi110.stdout:213: finishing write tid 2 to smithi11075562-17 2024-09-14T03:38:09.082 INFO:tasks.rados.rados.0.smithi110.stdout:213: finishing write tid 3 to smithi11075562-17 2024-09-14T03:38:09.082 INFO:tasks.rados.rados.0.smithi110.stdout:213: finishing write tid 4 to smithi11075562-17 2024-09-14T03:38:09.083 INFO:tasks.rados.rados.0.smithi110.stdout:213: finishing write tid 5 to smithi11075562-17 2024-09-14T03:38:09.083 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 24 (ObjNum 88 snap 15 seq_num 88) dirty exists 2024-09-14T03:38:09.084 INFO:tasks.rados.rados.0.smithi110.stdout:213: left oid 17 (ObjNum 88 snap 15 seq_num 88) 2024-09-14T03:38:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:09 smithi110 ceph-mon[27365]: pgmap v141: 121 pgs: 121 active+clean; 37 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T03:38:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:09 smithi110 ceph-mon[27365]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:38:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:09 smithi110 ceph-mon[35374]: pgmap v141: 121 pgs: 121 active+clean; 37 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T03:38:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:09 smithi110 ceph-mon[35374]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:38:09.440 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:38:09.440 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (2m) 109s ago 3m 18.3M - 0.25.0 c8568f914cd2 764997e49f35 2024-09-14T03:38:09.440 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (2m) 114s ago 2m 80.5M - 9.4.7 954c08fa6188 67ea5066f839 2024-09-14T03:38:09.440 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (4m) 109s ago 4m 16.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:38:09.440 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (4m) 114s ago 4m 15.0M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443 running (7m) 114s ago 7m 399M - 17.2.7-1629-geb16b97b ab51d1bebd12 68be7f1bb797 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:9283,8765 running (9m) 109s ago 9m 446M - 17.2.7-1629-geb16b97b ab51d1bebd12 a83ea41d9e62 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (9m) 109s ago 9m 44.8M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 c1073676c66c 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (8m) 114s ago 8m 40.7M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 b6415a0de807 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (7m) 109s ago 7m 39.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 8e2894672cd8 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (2m) 109s ago 2m 13.2M - 1.5.0 0da6a335fe13 986afca49027 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (2m) 114s ago 2m 3359k - 1.5.0 0da6a335fe13 67a66167cf52 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (7m) 109s ago 7m 43.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (7m) 109s ago 7m 43.8M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:38:09.441 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (6m) 109s ago 6m 43.1M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:38:09.442 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (6m) 109s ago 6m 41.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:38:09.442 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (5m) 114s ago 5m 41.7M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:38:09.442 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (5m) 114s ago 5m 42.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:38:09.442 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (5m) 114s ago 5m 41.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:38:09.442 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (4m) 114s ago 4m 42.8M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:38:09.442 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (2m) 114s ago 3m 24.8M - 2.43.0 a07b618ecd1d 4460d604b8f6 2024-09-14T03:38:09.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:09 smithi195 ceph-mon[30695]: pgmap v141: 121 pgs: 121 active+clean; 37 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T03:38:09.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:09 smithi195 ceph-mon[30695]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:38:09.744 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:38:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:38:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 15 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:38:09.745 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:38:10.002 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:38:10.002 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:38:10.002 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:38:10.002 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:38:10.002 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [], 2024-09-14T03:38:10.002 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "0/20 daemons upgraded", 2024-09-14T03:38:10.002 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 image on host smithi195", 2024-09-14T03:38:10.003 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:38:10.003 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:38:10.010 INFO:tasks.rados.rados.0.smithi110.stdout:213: done (1 left) 2024-09-14T03:38:10.010 INFO:tasks.rados.rados.0.smithi110.stdout:218: done (0 left) 2024-09-14T03:38:10.010 INFO:tasks.rados.rados.0.smithi110.stdout:219: copy_from oid 48 from oid 47 current snap is 16 2024-09-14T03:38:10.010 INFO:tasks.rados.rados.0.smithi110.stdout:220: read oid 34 snap -1 2024-09-14T03:38:10.010 INFO:tasks.rados.rados.0.smithi110.stdout:220: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-14T03:38:10.010 INFO:tasks.rados.rados.0.smithi110.stdout:221: snap_create 2024-09-14T03:38:10.014 INFO:tasks.rados.rados.0.smithi110.stdout:220: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-14T03:38:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:219: finishing copy_from racing read to smithi11075562-48 2024-09-14T03:38:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:219: finishing copy_from to smithi11075562-48 2024-09-14T03:38:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 51 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:38:10.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[27365]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[27365]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:38:10.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[27365]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[27365]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/10202410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:38:10.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[27365]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:38:10.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[35374]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[35374]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:38:10.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[35374]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[35374]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/10202410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:38:10.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-mon[35374]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:38:10.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:38:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:10 smithi195 ceph-mon[30695]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:10 smithi195 ceph-mon[30695]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:38:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:10 smithi195 ceph-mon[30695]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:10 smithi195 ceph-mon[30695]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:10 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/10202410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:38:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:10 smithi195 ceph-mon[30695]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:38:10.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:10 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:10] "GET /metrics HTTP/1.1" 200 239859 "" "Prometheus/2.43.0" 2024-09-14T03:38:11.011 INFO:tasks.rados.rados.0.smithi110.stdout:219: done (2 left) 2024-09-14T03:38:11.012 INFO:tasks.rados.rados.0.smithi110.stdout:220: done (1 left) 2024-09-14T03:38:11.012 INFO:tasks.rados.rados.0.smithi110.stdout:221: done (0 left) 2024-09-14T03:38:11.012 INFO:tasks.rados.rados.0.smithi110.stdout:222: read oid 47 snap -1 2024-09-14T03:38:11.012 INFO:tasks.rados.rados.0.smithi110.stdout:222: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:38:11.012 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 17 2024-09-14T03:38:11.012 INFO:tasks.rados.rados.0.smithi110.stdout:223: seq_num 90 ranges {24576=712704,737280=540672,1277952=262144} 2024-09-14T03:38:11.015 INFO:tasks.rados.rados.0.smithi110.stdout:223: writing smithi11075562-36 from 24576 to 737280 tid 1 2024-09-14T03:38:11.018 INFO:tasks.rados.rados.0.smithi110.stdout:223: writing smithi11075562-36 from 737280 to 1277952 tid 2 2024-09-14T03:38:11.020 INFO:tasks.rados.rados.0.smithi110.stdout:223: writing smithi11075562-36 from 1277952 to 1540096 tid 3 2024-09-14T03:38:11.020 INFO:tasks.rados.rados.0.smithi110.stdout:224: rollback oid 18 current snap is 17 2024-09-14T03:38:11.020 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 18 to 16 2024-09-14T03:38:11.020 INFO:tasks.rados.rados.0.smithi110.stdout:222: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:38:11.042 INFO:tasks.rados.rados.0.smithi110.stdout:223: finishing write tid 1 to smithi11075562-36 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:223: finishing write tid 2 to smithi11075562-36 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:224: finishing rollback tid 0 to smithi11075562-18 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:224: finishing rollback tid 1 to smithi11075562-18 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:223: finishing write tid 3 to smithi11075562-36 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:224: finishing rollback tid 2 to smithi11075562-18 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 53 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:223: finishing write tid 4 to smithi11075562-36 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:223: finishing write tid 5 to smithi11075562-36 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 70 (ObjNum 90 snap 17 seq_num 90) dirty exists 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:223: left oid 36 (ObjNum 90 snap 17 seq_num 90) 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:222: done (2 left) 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:223: done (1 left) 2024-09-14T03:38:11.043 INFO:tasks.rados.rados.0.smithi110.stdout:224: done (0 left) 2024-09-14T03:38:11.044 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 17 2024-09-14T03:38:11.044 INFO:tasks.rados.rados.0.smithi110.stdout:225: seq_num 91 ranges {0=729088,729088=425984,1155072=376832} 2024-09-14T03:38:11.046 INFO:tasks.rados.rados.0.smithi110.stdout:225: writing smithi11075562-2 from 0 to 729088 tid 1 2024-09-14T03:38:11.049 INFO:tasks.rados.rados.0.smithi110.stdout:225: writing smithi11075562-2 from 729088 to 1155072 tid 2 2024-09-14T03:38:11.051 INFO:tasks.rados.rados.0.smithi110.stdout:225: writing smithi11075562-2 from 1155072 to 1531904 tid 3 2024-09-14T03:38:11.051 INFO:tasks.rados.rados.0.smithi110.stdout:226: delete oid 40 current snap is 17 2024-09-14T03:38:11.051 INFO:tasks.rados.rados.0.smithi110.stdout:225: finishing write tid 1 to smithi11075562-2 2024-09-14T03:38:11.051 INFO:tasks.rados.rados.0.smithi110.stdout:225: finishing write tid 2 to smithi11075562-2 2024-09-14T03:38:11.053 INFO:tasks.rados.rados.0.smithi110.stdout:226: done (1 left) 2024-09-14T03:38:11.053 INFO:tasks.rados.rados.0.smithi110.stdout:227: read oid 47 snap -1 2024-09-14T03:38:11.053 INFO:tasks.rados.rados.0.smithi110.stdout:227: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:38:11.053 INFO:tasks.rados.rados.0.smithi110.stdout:228: rollback oid 49 current snap is 17 2024-09-14T03:38:11.053 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 13 2024-09-14T03:38:11.054 INFO:tasks.rados.rados.0.smithi110.stdout:229: snap_remove snap 14 2024-09-14T03:38:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:11 smithi110 ceph-mon[35374]: pgmap v144: 121 pgs: 121 active+clean; 37 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T03:38:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:11 smithi110 ceph-mon[35374]: from='client.24866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:11 smithi110 ceph-mon[35374]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:38:11.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:11 smithi110 ceph-mon[27365]: pgmap v144: 121 pgs: 121 active+clean; 37 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T03:38:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:11 smithi110 ceph-mon[27365]: from='client.24866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:11 smithi110 ceph-mon[27365]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:38:11.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:11 smithi195 ceph-mon[30695]: pgmap v144: 121 pgs: 121 active+clean; 37 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T03:38:11.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:11 smithi195 ceph-mon[30695]: from='client.24866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:11.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:11 smithi195 ceph-mon[30695]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:38:12.013 INFO:tasks.rados.rados.0.smithi110.stdout:229: done (3 left) 2024-09-14T03:38:12.013 INFO:tasks.rados.rados.0.smithi110.stdout:230: snap_create 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:225: finishing write tid 3 to smithi11075562-2 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:225: finishing write tid 4 to smithi11075562-2 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:225: finishing write tid 5 to smithi11075562-2 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 43 (ObjNum 91 snap 17 seq_num 91) dirty exists 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:225: left oid 2 (ObjNum 91 snap 17 seq_num 91) 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:228: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:228: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:228: finishing rollback tid 2 to smithi11075562-49 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 53 (ObjNum 87 snap 13 seq_num 87) dirty exists 2024-09-14T03:38:12.014 INFO:tasks.rados.rados.0.smithi110.stdout:227: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:38:13.015 INFO:tasks.rados.rados.0.smithi110.stdout:225: done (3 left) 2024-09-14T03:38:13.015 INFO:tasks.rados.rados.0.smithi110.stdout:227: done (2 left) 2024-09-14T03:38:13.015 INFO:tasks.rados.rados.0.smithi110.stdout:228: done (1 left) 2024-09-14T03:38:13.015 INFO:tasks.rados.rados.0.smithi110.stdout:230: done (0 left) 2024-09-14T03:38:13.015 INFO:tasks.rados.rados.0.smithi110.stdout:231: snap_create 2024-09-14T03:38:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:13 smithi110 ceph-mon[35374]: pgmap v147: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:38:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:13 smithi110 ceph-mon[35374]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:38:13.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:13 smithi110 ceph-mon[27365]: pgmap v147: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:38:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:13 smithi110 ceph-mon[27365]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:38:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:13 smithi195 ceph-mon[30695]: pgmap v147: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:38:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:13 smithi195 ceph-mon[30695]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:38:14.025 INFO:tasks.rados.rados.0.smithi110.stdout:231: done (0 left) 2024-09-14T03:38:14.025 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 19 2024-09-14T03:38:14.026 INFO:tasks.rados.rados.0.smithi110.stdout:232: seq_num 93 ranges {491520=761856,1253376=647168,1900544=155648} 2024-09-14T03:38:14.032 INFO:tasks.rados.rados.0.smithi110.stdout:232: writing smithi11075562-30 from 491520 to 1253376 tid 1 2024-09-14T03:38:14.035 INFO:tasks.rados.rados.0.smithi110.stdout:232: writing smithi11075562-30 from 1253376 to 1900544 tid 2 2024-09-14T03:38:14.037 INFO:tasks.rados.rados.0.smithi110.stdout:232: writing smithi11075562-30 from 1900544 to 2056192 tid 3 2024-09-14T03:38:14.037 INFO:tasks.rados.rados.0.smithi110.stdout:233: rmattr oid 17 current snap is 19 2024-09-14T03:38:14.037 INFO:tasks.rados.rados.0.smithi110.stdout:233: done (1 left) 2024-09-14T03:38:14.037 INFO:tasks.rados.rados.0.smithi110.stdout:234: snap_remove snap 12 2024-09-14T03:38:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:14 smithi110 ceph-mon[35374]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:38:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:14 smithi110 ceph-mon[27365]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:38:14.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:14 smithi195 ceph-mon[30695]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:38:15.013 INFO:tasks.rados.rados.0.smithi110.stdout:232: finishing write tid 1 to smithi11075562-30 2024-09-14T03:38:15.013 INFO:tasks.rados.rados.0.smithi110.stdout:232: finishing write tid 2 to smithi11075562-30 2024-09-14T03:38:15.013 INFO:tasks.rados.rados.0.smithi110.stdout:232: finishing write tid 3 to smithi11075562-30 2024-09-14T03:38:15.013 INFO:tasks.rados.rados.0.smithi110.stdout:232: finishing write tid 4 to smithi11075562-30 2024-09-14T03:38:15.013 INFO:tasks.rados.rados.0.smithi110.stdout:232: finishing write tid 5 to smithi11075562-30 2024-09-14T03:38:15.013 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 42 (ObjNum 93 snap 19 seq_num 93) dirty exists 2024-09-14T03:38:15.014 INFO:tasks.rados.rados.0.smithi110.stdout:232: left oid 30 (ObjNum 93 snap 19 seq_num 93) 2024-09-14T03:38:15.014 INFO:tasks.rados.rados.0.smithi110.stdout:232: done (1 left) 2024-09-14T03:38:15.014 INFO:tasks.rados.rados.0.smithi110.stdout:234: done (0 left) 2024-09-14T03:38:15.014 INFO:tasks.rados.rados.0.smithi110.stdout:235: snap_remove snap 15 2024-09-14T03:38:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:15 smithi110 ceph-mon[27365]: pgmap v150: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:38:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:15 smithi110 ceph-mon[27365]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:38:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:15 smithi110 ceph-mon[27365]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:38:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:15 smithi110 ceph-mon[35374]: pgmap v150: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:38:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:15 smithi110 ceph-mon[35374]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:38:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:15 smithi110 ceph-mon[35374]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:38:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:15 smithi195 ceph-mon[30695]: pgmap v150: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:38:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:15 smithi195 ceph-mon[30695]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:38:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:15 smithi195 ceph-mon[30695]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:38:16.013 INFO:tasks.rados.rados.0.smithi110.stdout:235: done (0 left) 2024-09-14T03:38:16.013 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 19 2024-09-14T03:38:16.014 INFO:tasks.rados.rados.0.smithi110.stdout:236: seq_num 94 ranges {221184=778240,999424=532480,1531904=270336} 2024-09-14T03:38:16.019 INFO:tasks.rados.rados.0.smithi110.stdout:236: writing smithi11075562-14 from 221184 to 999424 tid 1 2024-09-14T03:38:16.022 INFO:tasks.rados.rados.0.smithi110.stdout:236: writing smithi11075562-14 from 999424 to 1531904 tid 2 2024-09-14T03:38:16.024 INFO:tasks.rados.rados.0.smithi110.stdout:236: writing smithi11075562-14 from 1531904 to 1802240 tid 3 2024-09-14T03:38:16.025 INFO:tasks.rados.rados.0.smithi110.stdout:237: read oid 3 snap -1 2024-09-14T03:38:16.025 INFO:tasks.rados.rados.0.smithi110.stdout:237: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:38:16.025 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 19 2024-09-14T03:38:16.025 INFO:tasks.rados.rados.0.smithi110.stdout:238: seq_num 95 ranges {696320=794624,1490944=409600,1900544=393216} 2024-09-14T03:38:16.033 INFO:tasks.rados.rados.0.smithi110.stdout:238: writing smithi11075562-20 from 696320 to 1490944 tid 1 2024-09-14T03:38:16.035 INFO:tasks.rados.rados.0.smithi110.stdout:238: writing smithi11075562-20 from 1490944 to 1900544 tid 2 2024-09-14T03:38:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:238: writing smithi11075562-20 from 1900544 to 2293760 tid 3 2024-09-14T03:38:16.038 INFO:tasks.rados.rados.0.smithi110.stdout:239: snap_create 2024-09-14T03:38:16.038 INFO:tasks.rados.rados.0.smithi110.stdout:236: finishing write tid 1 to smithi11075562-14 2024-09-14T03:38:16.038 INFO:tasks.rados.rados.0.smithi110.stdout:236: finishing write tid 2 to smithi11075562-14 2024-09-14T03:38:16.038 INFO:tasks.rados.rados.0.smithi110.stdout:236: finishing write tid 3 to smithi11075562-14 2024-09-14T03:38:16.038 INFO:tasks.rados.rados.0.smithi110.stdout:237: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:38:16.050 INFO:tasks.rados.rados.0.smithi110.stdout:236: finishing write tid 4 to smithi11075562-14 2024-09-14T03:38:16.050 INFO:tasks.rados.rados.0.smithi110.stdout:236: finishing write tid 5 to smithi11075562-14 2024-09-14T03:38:16.050 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 59 (ObjNum 94 snap 19 seq_num 94) dirty exists 2024-09-14T03:38:16.050 INFO:tasks.rados.rados.0.smithi110.stdout:236: left oid 14 (ObjNum 94 snap 19 seq_num 94) 2024-09-14T03:38:16.050 INFO:tasks.rados.rados.0.smithi110.stdout:238: finishing write tid 1 to smithi11075562-20 2024-09-14T03:38:16.050 INFO:tasks.rados.rados.0.smithi110.stdout:238: finishing write tid 2 to smithi11075562-20 2024-09-14T03:38:16.051 INFO:tasks.rados.rados.0.smithi110.stdout:238: finishing write tid 3 to smithi11075562-20 2024-09-14T03:38:16.051 INFO:tasks.rados.rados.0.smithi110.stdout:238: finishing write tid 4 to smithi11075562-20 2024-09-14T03:38:16.051 INFO:tasks.rados.rados.0.smithi110.stdout:238: finishing write tid 5 to smithi11075562-20 2024-09-14T03:38:16.051 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 65 (ObjNum 95 snap 19 seq_num 95) dirty exists 2024-09-14T03:38:16.051 INFO:tasks.rados.rados.0.smithi110.stdout:238: left oid 20 (ObjNum 95 snap 19 seq_num 95) 2024-09-14T03:38:17.027 INFO:tasks.rados.rados.0.smithi110.stdout:236: done (3 left) 2024-09-14T03:38:17.027 INFO:tasks.rados.rados.0.smithi110.stdout:237: done (2 left) 2024-09-14T03:38:17.028 INFO:tasks.rados.rados.0.smithi110.stdout:238: done (1 left) 2024-09-14T03:38:17.028 INFO:tasks.rados.rados.0.smithi110.stdout:239: done (0 left) 2024-09-14T03:38:17.028 INFO:tasks.rados.rados.0.smithi110.stdout:240: delete oid 41 current snap is 20 2024-09-14T03:38:17.030 INFO:tasks.rados.rados.0.smithi110.stdout:240: done (0 left) 2024-09-14T03:38:17.030 INFO:tasks.rados.rados.0.smithi110.stdout:241: read oid 13 snap -1 2024-09-14T03:38:17.030 INFO:tasks.rados.rados.0.smithi110.stdout:241: expect (ObjNum 80 snap 13 seq_num 80) 2024-09-14T03:38:17.030 INFO:tasks.rados.rados.0.smithi110.stdout:242: setattr oid 26 current snap is 20 2024-09-14T03:38:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:243: copy_from oid 32 from oid 7 current snap is 20 2024-09-14T03:38:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:244: read oid 22 snap -1 2024-09-14T03:38:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:244: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-14T03:38:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 20 2024-09-14T03:38:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:245: seq_num 97 ranges {1024000=434176,1458176=581632,2039808=540672,2580480=81920} 2024-09-14T03:38:17.042 INFO:tasks.rados.rados.0.smithi110.stdout:245: writing smithi11075562-15 from 1024000 to 1458176 tid 1 2024-09-14T03:38:17.045 INFO:tasks.rados.rados.0.smithi110.stdout:245: writing smithi11075562-15 from 1458176 to 2039808 tid 2 2024-09-14T03:38:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:245: writing smithi11075562-15 from 2039808 to 2580480 tid 3 2024-09-14T03:38:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:245: writing smithi11075562-15 from 2580480 to 2662400 tid 4 2024-09-14T03:38:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 20 2024-09-14T03:38:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:246: seq_num 98 ranges {2277376=450560,2727936=466944,3194880=737280} 2024-09-14T03:38:17.064 INFO:tasks.rados.rados.0.smithi110.stdout:246: writing smithi11075562-49 from 2277376 to 2727936 tid 1 2024-09-14T03:38:17.067 INFO:tasks.rados.rados.0.smithi110.stdout:246: writing smithi11075562-49 from 2727936 to 3194880 tid 2 2024-09-14T03:38:17.071 INFO:tasks.rados.rados.0.smithi110.stdout:246: writing smithi11075562-49 from 3194880 to 3932160 tid 3 2024-09-14T03:38:17.072 INFO:tasks.rados.rados.0.smithi110.stdout:247: rollback oid 33 current snap is 20 2024-09-14T03:38:17.072 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 33 to 16 2024-09-14T03:38:17.072 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 11 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:38:17.072 INFO:tasks.rados.rados.0.smithi110.stdout:244: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-14T03:38:17.081 INFO:tasks.rados.rados.0.smithi110.stdout:243: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:38:17.082 INFO:tasks.rados.rados.0.smithi110.stdout:241: expect (ObjNum 80 snap 13 seq_num 80) 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:243: finishing copy_from to smithi11075562-32 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:245: finishing write tid 1 to smithi11075562-15 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:241: done (6 left) 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:242: done (5 left) 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:243: done (4 left) 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:244: done (3 left) 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 20 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:245: finishing write tid 2 to smithi11075562-15 2024-09-14T03:38:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:245: finishing write tid 3 to smithi11075562-15 2024-09-14T03:38:17.126 INFO:tasks.rados.rados.0.smithi110.stdout:248: seq_num 99 ranges {606208=466944,1073152=802816,1875968=401408} 2024-09-14T03:38:17.130 INFO:tasks.rados.rados.0.smithi110.stdout:248: writing smithi11075562-25 from 606208 to 1073152 tid 1 2024-09-14T03:38:17.134 INFO:tasks.rados.rados.0.smithi110.stdout:248: writing smithi11075562-25 from 1073152 to 1875968 tid 2 2024-09-14T03:38:17.137 INFO:tasks.rados.rados.0.smithi110.stdout:248: writing smithi11075562-25 from 1875968 to 2277376 tid 3 2024-09-14T03:38:17.137 INFO:tasks.rados.rados.0.smithi110.stdout:249: read oid 5 snap -1 2024-09-14T03:38:17.137 INFO:tasks.rados.rados.0.smithi110.stdout:249: expect deleted 2024-09-14T03:38:17.137 INFO:tasks.rados.rados.0.smithi110.stdout:250: rollback oid 28 current snap is 20 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 28 to 16 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:245: finishing write tid 4 to smithi11075562-15 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:245: finishing write tid 5 to smithi11075562-15 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:245: finishing write tid 6 to smithi11075562-15 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 54 (ObjNum 97 snap 20 seq_num 97) dirty exists 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:245: left oid 15 (ObjNum 97 snap 20 seq_num 97) 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:246: finishing write tid 1 to smithi11075562-49 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:245: done (5 left) 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:251: rmattr oid 41 current snap is 20 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:251: done (5 left) 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:252: copy_from oid 45 from oid 14 current snap is 20 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:246: finishing write tid 2 to smithi11075562-49 2024-09-14T03:38:17.138 INFO:tasks.rados.rados.0.smithi110.stdout:246: finishing write tid 3 to smithi11075562-49 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:246: finishing write tid 4 to smithi11075562-49 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:246: finishing write tid 5 to smithi11075562-49 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 68 (ObjNum 98 snap 20 seq_num 98) dirty exists 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:246: left oid 49 (ObjNum 98 snap 20 seq_num 98) 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:247: finishing rollback tid 0 to smithi11075562-33 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:247: finishing rollback tid 1 to smithi11075562-33 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:247: finishing rollback tid 2 to smithi11075562-33 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 69 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:248: finishing write tid 1 to smithi11075562-25 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:246: done (5 left) 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:247: done (4 left) 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 20 2024-09-14T03:38:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:253: seq_num 100 ranges {106496=483328,589824=688128,1277952=516096} 2024-09-14T03:38:17.140 INFO:tasks.rados.rados.0.smithi110.stdout:253: writing smithi11075562-32 from 106496 to 589824 tid 1 2024-09-14T03:38:17.145 INFO:tasks.rados.rados.0.smithi110.stdout:253: writing smithi11075562-32 from 589824 to 1277952 tid 2 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:253: writing smithi11075562-32 from 1277952 to 1794048 tid 3 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 20 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:248: finishing write tid 2 to smithi11075562-25 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:250: finishing rollback tid 0 to smithi11075562-28 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:248: finishing write tid 3 to smithi11075562-25 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:250: finishing rollback tid 1 to smithi11075562-28 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:250: finishing rollback tid 2 to smithi11075562-28 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 51 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:38:17.150 INFO:tasks.rados.rados.0.smithi110.stdout:248: finishing write tid 4 to smithi11075562-25 2024-09-14T03:38:17.151 INFO:tasks.rados.rados.0.smithi110.stdout:248: finishing write tid 5 to smithi11075562-25 2024-09-14T03:38:17.151 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 83 (ObjNum 99 snap 20 seq_num 99) dirty exists 2024-09-14T03:38:17.151 INFO:tasks.rados.rados.0.smithi110.stdout:248: left oid 25 (ObjNum 99 snap 20 seq_num 99) 2024-09-14T03:38:17.151 INFO:tasks.rados.rados.0.smithi110.stdout:253: finishing write tid 1 to smithi11075562-32 2024-09-14T03:38:17.151 INFO:tasks.rados.rados.0.smithi110.stdout:253: finishing write tid 2 to smithi11075562-32 2024-09-14T03:38:17.151 INFO:tasks.rados.rados.0.smithi110.stdout:254: seq_num 101 ranges {0=499712,499712=573440,1073152=524288,1597440=106496} 2024-09-14T03:38:17.151 INFO:tasks.rados.rados.0.smithi110.stdout:254: writing smithi11075562-11 from 0 to 499712 tid 1 2024-09-14T03:38:17.154 INFO:tasks.rados.rados.0.smithi110.stdout:254: writing smithi11075562-11 from 499712 to 1073152 tid 2 2024-09-14T03:38:17.157 INFO:tasks.rados.rados.0.smithi110.stdout:254: writing smithi11075562-11 from 1073152 to 1597440 tid 3 2024-09-14T03:38:17.158 INFO:tasks.rados.rados.0.smithi110.stdout:254: writing smithi11075562-11 from 1597440 to 1703936 tid 4 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:248: done (5 left) 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:249: done (4 left) 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:250: done (3 left) 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:255: read oid 39 snap 18 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:255: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:256: rmattr oid 49 current snap is 20 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:252: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:253: finishing write tid 3 to smithi11075562-32 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:253: finishing write tid 4 to smithi11075562-32 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:252: finishing copy_from to smithi11075562-45 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 59 (ObjNum 94 snap 19 seq_num 94) dirty exists 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:253: finishing write tid 5 to smithi11075562-32 2024-09-14T03:38:17.159 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 58 (ObjNum 100 snap 20 seq_num 100) dirty exists 2024-09-14T03:38:17.160 INFO:tasks.rados.rados.0.smithi110.stdout:253: left oid 32 (ObjNum 100 snap 20 seq_num 100) 2024-09-14T03:38:17.160 INFO:tasks.rados.rados.0.smithi110.stdout:254: finishing write tid 1 to smithi11075562-11 2024-09-14T03:38:17.160 INFO:tasks.rados.rados.0.smithi110.stdout:254: finishing write tid 2 to smithi11075562-11 2024-09-14T03:38:17.160 INFO:tasks.rados.rados.0.smithi110.stdout:252: done (4 left) 2024-09-14T03:38:17.160 INFO:tasks.rados.rados.0.smithi110.stdout:253: done (3 left) 2024-09-14T03:38:17.160 INFO:tasks.rados.rados.0.smithi110.stdout:256: done (2 left) 2024-09-14T03:38:17.160 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 20 2024-09-14T03:38:17.160 INFO:tasks.rados.rados.0.smithi110.stdout:257: seq_num 102 ranges {2752512=516096,3268608=450560,3719168=729088,4448256=24576} 2024-09-14T03:38:17.175 INFO:tasks.rados.rados.0.smithi110.stdout:257: writing smithi11075562-38 from 2752512 to 3268608 tid 1 2024-09-14T03:38:17.178 INFO:tasks.rados.rados.0.smithi110.stdout:257: writing smithi11075562-38 from 3268608 to 3719168 tid 2 2024-09-14T03:38:17.182 INFO:tasks.rados.rados.0.smithi110.stdout:257: writing smithi11075562-38 from 3719168 to 4448256 tid 3 2024-09-14T03:38:17.183 INFO:tasks.rados.rados.0.smithi110.stdout:257: writing smithi11075562-38 from 4448256 to 4472832 tid 4 2024-09-14T03:38:17.183 INFO:tasks.rados.rados.0.smithi110.stdout:258: read oid 35 snap -1 2024-09-14T03:38:17.183 INFO:tasks.rados.rados.0.smithi110.stdout:258: expect deleted 2024-09-14T03:38:17.183 INFO:tasks.rados.rados.0.smithi110.stdout:259: read oid 33 snap 17 2024-09-14T03:38:17.183 INFO:tasks.rados.rados.0.smithi110.stdout:259: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-14T03:38:17.183 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 20 2024-09-14T03:38:17.183 INFO:tasks.rados.rados.0.smithi110.stdout:260: seq_num 103 ranges {606208=532480,1138688=737280,1875968=466944} 2024-09-14T03:38:17.189 INFO:tasks.rados.rados.0.smithi110.stdout:260: writing smithi11075562-37 from 606208 to 1138688 tid 1 2024-09-14T03:38:17.193 INFO:tasks.rados.rados.0.smithi110.stdout:260: writing smithi11075562-37 from 1138688 to 1875968 tid 2 2024-09-14T03:38:17.196 INFO:tasks.rados.rados.0.smithi110.stdout:260: writing smithi11075562-37 from 1875968 to 2342912 tid 3 2024-09-14T03:38:17.196 INFO:tasks.rados.rados.0.smithi110.stdout:261: snap_remove snap 19 2024-09-14T03:38:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:17 smithi110 ceph-mon[35374]: pgmap v153: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:17 smithi110 ceph-mon[35374]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:38:17.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:17 smithi110 ceph-mon[27365]: pgmap v153: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:17 smithi110 ceph-mon[27365]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:38:17.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:17 smithi195 ceph-mon[30695]: pgmap v153: 121 pgs: 121 active+clean; 46 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:17.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:17 smithi195 ceph-mon[30695]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:38:18.033 INFO:tasks.rados.rados.0.smithi110.stdout:261: done (6 left) 2024-09-14T03:38:18.034 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 20 2024-09-14T03:38:18.034 INFO:tasks.rados.rados.0.smithi110.stdout:262: seq_num 104 ranges {1540096=548864,2088960=622592,2711552=581632} 2024-09-14T03:38:18.045 INFO:tasks.rados.rados.0.smithi110.stdout:262: writing smithi11075562-36 from 1540096 to 2088960 tid 1 2024-09-14T03:38:18.048 INFO:tasks.rados.rados.0.smithi110.stdout:262: writing smithi11075562-36 from 2088960 to 2711552 tid 2 2024-09-14T03:38:18.052 INFO:tasks.rados.rados.0.smithi110.stdout:262: writing smithi11075562-36 from 2711552 to 3293184 tid 3 2024-09-14T03:38:18.052 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 20 2024-09-14T03:38:18.052 INFO:tasks.rados.rados.0.smithi110.stdout:255: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-14T03:38:18.063 INFO:tasks.rados.rados.0.smithi110.stdout:254: finishing write tid 3 to smithi11075562-11 2024-09-14T03:38:18.063 INFO:tasks.rados.rados.0.smithi110.stdout:263: seq_num 105 ranges {360448=565248,925696=507904,1433600=507904,1941504=188416} 2024-09-14T03:38:18.068 INFO:tasks.rados.rados.0.smithi110.stdout:263: writing smithi11075562-22 from 360448 to 925696 tid 1 2024-09-14T03:38:18.071 INFO:tasks.rados.rados.0.smithi110.stdout:263: writing smithi11075562-22 from 925696 to 1433600 tid 2 2024-09-14T03:38:18.074 INFO:tasks.rados.rados.0.smithi110.stdout:263: writing smithi11075562-22 from 1433600 to 1941504 tid 3 2024-09-14T03:38:18.077 INFO:tasks.rados.rados.0.smithi110.stdout:263: writing smithi11075562-22 from 1941504 to 2129920 tid 4 2024-09-14T03:38:18.077 INFO:tasks.rados.rados.0.smithi110.stdout:255: done (7 left) 2024-09-14T03:38:18.077 INFO:tasks.rados.rados.0.smithi110.stdout:264: rollback oid 24 current snap is 20 2024-09-14T03:38:18.077 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 17 2024-09-14T03:38:18.077 INFO:tasks.rados.rados.0.smithi110.stdout:254: finishing write tid 4 to smithi11075562-11 2024-09-14T03:38:18.077 INFO:tasks.rados.rados.0.smithi110.stdout:254: finishing write tid 5 to smithi11075562-11 2024-09-14T03:38:18.077 INFO:tasks.rados.rados.0.smithi110.stdout:254: finishing write tid 6 to smithi11075562-11 2024-09-14T03:38:18.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 56 (ObjNum 101 snap 20 seq_num 101) dirty exists 2024-09-14T03:38:18.078 INFO:tasks.rados.rados.0.smithi110.stdout:254: left oid 11 (ObjNum 101 snap 20 seq_num 101) 2024-09-14T03:38:18.078 INFO:tasks.rados.rados.0.smithi110.stdout:257: finishing write tid 1 to smithi11075562-38 2024-09-14T03:38:18.078 INFO:tasks.rados.rados.0.smithi110.stdout:257: finishing write tid 2 to smithi11075562-38 2024-09-14T03:38:18.078 INFO:tasks.rados.rados.0.smithi110.stdout:254: done (7 left) 2024-09-14T03:38:18.078 INFO:tasks.rados.rados.0.smithi110.stdout:265: rollback oid 20 current snap is 20 2024-09-14T03:38:18.078 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 16 2024-09-14T03:38:18.079 INFO:tasks.rados.rados.0.smithi110.stdout:257: finishing write tid 3 to smithi11075562-38 2024-09-14T03:38:18.079 INFO:tasks.rados.rados.0.smithi110.stdout:257: finishing write tid 4 to smithi11075562-38 2024-09-14T03:38:18.079 INFO:tasks.rados.rados.0.smithi110.stdout:257: finishing write tid 5 to smithi11075562-38 2024-09-14T03:38:18.079 INFO:tasks.rados.rados.0.smithi110.stdout:259: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-14T03:38:18.089 INFO:tasks.rados.rados.0.smithi110.stdout:257: finishing write tid 6 to smithi11075562-38 2024-09-14T03:38:18.089 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 47 (ObjNum 102 snap 20 seq_num 102) dirty exists 2024-09-14T03:38:18.089 INFO:tasks.rados.rados.0.smithi110.stdout:257: left oid 38 (ObjNum 102 snap 20 seq_num 102) 2024-09-14T03:38:18.089 INFO:tasks.rados.rados.0.smithi110.stdout:260: finishing write tid 1 to smithi11075562-37 2024-09-14T03:38:18.089 INFO:tasks.rados.rados.0.smithi110.stdout:257: done (7 left) 2024-09-14T03:38:18.089 INFO:tasks.rados.rados.0.smithi110.stdout:258: done (6 left) 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:259: done (5 left) 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:266: rmattr oid 23 current snap is 20 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:266: done (5 left) 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:267: rollback oid 21 current snap is 20 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 17 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:260: finishing write tid 2 to smithi11075562-37 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:260: finishing write tid 3 to smithi11075562-37 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:260: finishing write tid 4 to smithi11075562-37 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:260: finishing write tid 5 to smithi11075562-37 2024-09-14T03:38:18.090 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 90 (ObjNum 103 snap 20 seq_num 103) dirty exists 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:260: left oid 37 (ObjNum 103 snap 20 seq_num 103) 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:262: finishing write tid 1 to smithi11075562-36 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:262: finishing write tid 2 to smithi11075562-36 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:262: finishing write tid 3 to smithi11075562-36 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:262: finishing write tid 4 to smithi11075562-36 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:262: finishing write tid 5 to smithi11075562-36 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 101 (ObjNum 104 snap 20 seq_num 104) dirty exists 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:262: left oid 36 (ObjNum 104 snap 20 seq_num 104) 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:263: finishing write tid 1 to smithi11075562-22 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:263: finishing write tid 2 to smithi11075562-22 2024-09-14T03:38:18.091 INFO:tasks.rados.rados.0.smithi110.stdout:265: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:263: finishing write tid 3 to smithi11075562-22 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:264: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:263: finishing write tid 4 to smithi11075562-22 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:265: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:264: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:263: finishing write tid 5 to smithi11075562-22 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:265: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 74 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:264: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 43 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-14T03:38:18.092 INFO:tasks.rados.rados.0.smithi110.stdout:263: finishing write tid 6 to smithi11075562-22 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 68 (ObjNum 105 snap 20 seq_num 105) dirty exists 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:263: left oid 22 (ObjNum 105 snap 20 seq_num 105) 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:260: done (5 left) 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:262: done (4 left) 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:263: done (3 left) 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:264: done (2 left) 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:265: done (1 left) 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:268: rmattr oid 47 current snap is 20 2024-09-14T03:38:18.093 INFO:tasks.rados.rados.0.smithi110.stdout:269: copy_from oid 35 from oid 28 current snap is 20 2024-09-14T03:38:18.094 INFO:tasks.rados.rados.0.smithi110.stdout:270: rollback oid 39 current snap is 20 2024-09-14T03:38:18.094 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 17 2024-09-14T03:38:18.094 INFO:tasks.rados.rados.0.smithi110.stdout:271: snap_remove snap 18 2024-09-14T03:38:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:18 smithi110 ceph-mon[35374]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:38:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:18 smithi110 ceph-mon[27365]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:38:18.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:18 smithi195 ceph-mon[30695]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:271: done (4 left) 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:272: snap_create 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 103 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:269: finishing copy_from racing read to smithi11075562-35 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:267: finishing rollback tid 0 to smithi11075562-21 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:269: finishing copy_from to smithi11075562-35 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 51 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:267: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:267: finishing rollback tid 2 to smithi11075562-21 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 64 (ObjNum 81 snap 13 seq_num 81) dirty exists 2024-09-14T03:38:19.038 INFO:tasks.rados.rados.0.smithi110.stdout:270: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:38:19.039 INFO:tasks.rados.rados.0.smithi110.stdout:270: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:38:19.039 INFO:tasks.rados.rados.0.smithi110.stdout:270: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:38:19.039 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 68 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-14T03:38:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:19 smithi110 ceph-mon[35374]: pgmap v156: 121 pgs: 121 active+clean; 44 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:38:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:19 smithi110 ceph-mon[35374]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:38:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:19 smithi110 ceph-mon[27365]: pgmap v156: 121 pgs: 121 active+clean; 44 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:38:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:19 smithi110 ceph-mon[27365]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:38:19.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:19 smithi195 ceph-mon[30695]: pgmap v156: 121 pgs: 121 active+clean; 44 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:38:19.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:19 smithi195 ceph-mon[30695]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:38:20.014 INFO:tasks.rados.rados.0.smithi110.stdout:267: done (4 left) 2024-09-14T03:38:20.014 INFO:tasks.rados.rados.0.smithi110.stdout:268: done (3 left) 2024-09-14T03:38:20.014 INFO:tasks.rados.rados.0.smithi110.stdout:269: done (2 left) 2024-09-14T03:38:20.014 INFO:tasks.rados.rados.0.smithi110.stdout:270: done (1 left) 2024-09-14T03:38:20.014 INFO:tasks.rados.rados.0.smithi110.stdout:272: done (0 left) 2024-09-14T03:38:20.014 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 21 2024-09-14T03:38:20.014 INFO:tasks.rados.rados.0.smithi110.stdout:273: seq_num 106 ranges {1417216=581632,1998848=794624,2793472=409600} 2024-09-14T03:38:20.024 INFO:tasks.rados.rados.0.smithi110.stdout:273: writing smithi11075562-16 from 1417216 to 1998848 tid 1 2024-09-14T03:38:20.029 INFO:tasks.rados.rados.0.smithi110.stdout:273: writing smithi11075562-16 from 1998848 to 2793472 tid 2 2024-09-14T03:38:20.032 INFO:tasks.rados.rados.0.smithi110.stdout:273: writing smithi11075562-16 from 2793472 to 3203072 tid 3 2024-09-14T03:38:20.032 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 21 2024-09-14T03:38:20.032 INFO:tasks.rados.rados.0.smithi110.stdout:274: seq_num 107 ranges {761856=598016,1359872=729088,2088960=466944,2555904=8192} 2024-09-14T03:38:20.039 INFO:tasks.rados.rados.0.smithi110.stdout:274: writing smithi11075562-18 from 761856 to 1359872 tid 1 2024-09-14T03:38:20.043 INFO:tasks.rados.rados.0.smithi110.stdout:274: writing smithi11075562-18 from 1359872 to 2088960 tid 2 2024-09-14T03:38:20.046 INFO:tasks.rados.rados.0.smithi110.stdout:274: writing smithi11075562-18 from 2088960 to 2555904 tid 3 2024-09-14T03:38:20.046 INFO:tasks.rados.rados.0.smithi110.stdout:274: writing smithi11075562-18 from 2555904 to 2564096 tid 4 2024-09-14T03:38:20.046 INFO:tasks.rados.rados.0.smithi110.stdout:275: setattr oid 45 current snap is 21 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:273: finishing write tid 1 to smithi11075562-16 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:273: finishing write tid 2 to smithi11075562-16 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:273: finishing write tid 3 to smithi11075562-16 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:273: finishing write tid 4 to smithi11075562-16 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:273: finishing write tid 5 to smithi11075562-16 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 72 (ObjNum 106 snap 21 seq_num 106) dirty exists 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:273: left oid 16 (ObjNum 106 snap 21 seq_num 106) 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:274: finishing write tid 1 to smithi11075562-18 2024-09-14T03:38:20.047 INFO:tasks.rados.rados.0.smithi110.stdout:274: finishing write tid 2 to smithi11075562-18 2024-09-14T03:38:20.049 INFO:tasks.rados.rados.0.smithi110.stdout:273: done (2 left) 2024-09-14T03:38:20.049 INFO:tasks.rados.rados.0.smithi110.stdout:276: snap_remove snap 13 2024-09-14T03:38:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:20 smithi110 ceph-mon[35374]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:38:20.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:20 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:20.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:20 smithi110 ceph-mon[35374]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:38:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:20 smithi110 ceph-mon[27365]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:38:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:20 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:20 smithi110 ceph-mon[27365]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:38:20.461 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:38:20.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:20 smithi195 ceph-mon[30695]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:38:20.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:20 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:20.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:20 smithi195 ceph-mon[30695]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:38:20.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:20] "GET /metrics HTTP/1.1" 200 239895 "" "Prometheus/2.43.0" 2024-09-14T03:38:21.050 INFO:tasks.rados.rados.0.smithi110.stdout:276: done (2 left) 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:277: setattr oid 15 current snap is 21 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:274: finishing write tid 3 to smithi11075562-18 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:274: finishing write tid 4 to smithi11075562-18 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:274: finishing write tid 5 to smithi11075562-18 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:274: finishing write tid 6 to smithi11075562-18 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 78 (ObjNum 107 snap 21 seq_num 107) dirty exists 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:274: left oid 18 (ObjNum 107 snap 21 seq_num 107) 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 115 (ObjNum 94 snap 19 seq_num 94) dirty exists 2024-09-14T03:38:21.051 INFO:tasks.rados.rados.0.smithi110.stdout:274: done (2 left) 2024-09-14T03:38:21.052 INFO:tasks.rados.rados.0.smithi110.stdout:275: done (1 left) 2024-09-14T03:38:21.052 INFO:tasks.rados.rados.0.smithi110.stdout:278: snap_create 2024-09-14T03:38:21.055 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 82 (ObjNum 97 snap 20 seq_num 97) dirty exists 2024-09-14T03:38:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:21 smithi110 ceph-mon[27365]: pgmap v159: 121 pgs: 121 active+clean; 44 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:38:21.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:21 smithi110 ceph-mon[35374]: pgmap v159: 121 pgs: 121 active+clean; 44 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:38:21.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:21 smithi195 ceph-mon[30695]: pgmap v159: 121 pgs: 121 active+clean; 44 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:38:22.052 INFO:tasks.rados.rados.0.smithi110.stdout:277: done (1 left) 2024-09-14T03:38:22.052 INFO:tasks.rados.rados.0.smithi110.stdout:278: done (0 left) 2024-09-14T03:38:22.052 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 22 2024-09-14T03:38:22.052 INFO:tasks.rados.rados.0.smithi110.stdout:279: seq_num 108 ranges {360448=614400,974848=606208,1581056=598016} 2024-09-14T03:38:22.057 INFO:tasks.rados.rados.0.smithi110.stdout:279: writing smithi11075562-33 from 360448 to 974848 tid 1 2024-09-14T03:38:22.060 INFO:tasks.rados.rados.0.smithi110.stdout:279: writing smithi11075562-33 from 974848 to 1581056 tid 2 2024-09-14T03:38:22.065 INFO:tasks.rados.rados.0.smithi110.stdout:279: writing smithi11075562-33 from 1581056 to 2179072 tid 3 2024-09-14T03:38:22.065 INFO:tasks.rados.rados.0.smithi110.stdout:280: copy_from oid 18 from oid 13 current snap is 22 2024-09-14T03:38:22.065 INFO:tasks.rados.rados.0.smithi110.stdout:279: finishing write tid 1 to smithi11075562-33 2024-09-14T03:38:22.065 INFO:tasks.rados.rados.0.smithi110.stdout:279: finishing write tid 2 to smithi11075562-33 2024-09-14T03:38:22.065 INFO:tasks.rados.rados.0.smithi110.stdout:281: snap_create 2024-09-14T03:38:22.067 INFO:tasks.rados.rados.0.smithi110.stdout:279: finishing write tid 3 to smithi11075562-33 2024-09-14T03:38:22.067 INFO:tasks.rados.rados.0.smithi110.stdout:279: finishing write tid 4 to smithi11075562-33 2024-09-14T03:38:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:279: finishing write tid 5 to smithi11075562-33 2024-09-14T03:38:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 87 (ObjNum 108 snap 22 seq_num 108) dirty exists 2024-09-14T03:38:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:279: left oid 33 (ObjNum 108 snap 22 seq_num 108) 2024-09-14T03:38:22.072 INFO:tasks.rados.rados.0.smithi110.stdout:280: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:38:22.073 INFO:tasks.rados.rados.0.smithi110.stdout:280: finishing copy_from to smithi11075562-18 2024-09-14T03:38:22.074 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 38 (ObjNum 80 snap 13 seq_num 80) dirty exists 2024-09-14T03:38:22.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:22 smithi110 ceph-mon[27365]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:38:22.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:22 smithi110 ceph-mon[35374]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:38:22.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:22 smithi195 ceph-mon[30695]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:38:23.053 INFO:tasks.rados.rados.0.smithi110.stdout:279: done (2 left) 2024-09-14T03:38:23.053 INFO:tasks.rados.rados.0.smithi110.stdout:280: done (1 left) 2024-09-14T03:38:23.053 INFO:tasks.rados.rados.0.smithi110.stdout:281: done (0 left) 2024-09-14T03:38:23.053 INFO:tasks.rados.rados.0.smithi110.stdout:282: snap_create 2024-09-14T03:38:23.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:23 smithi110 ceph-mon[27365]: pgmap v162: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 5.7 MiB/s wr, 39 op/s 2024-09-14T03:38:23.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:23 smithi110 ceph-mon[27365]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:38:23.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:23 smithi110 ceph-mon[35374]: pgmap v162: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 5.7 MiB/s wr, 39 op/s 2024-09-14T03:38:23.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:23 smithi110 ceph-mon[35374]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:38:23.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:23 smithi195 ceph-mon[30695]: pgmap v162: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 5.7 MiB/s wr, 39 op/s 2024-09-14T03:38:23.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:23 smithi195 ceph-mon[30695]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:38:24.062 INFO:tasks.rados.rados.0.smithi110.stdout:282: done (0 left) 2024-09-14T03:38:24.062 INFO:tasks.rados.rados.0.smithi110.stdout:283: read oid 15 snap -1 2024-09-14T03:38:24.062 INFO:tasks.rados.rados.0.smithi110.stdout:283: expect (ObjNum 97 snap 20 seq_num 97) 2024-09-14T03:38:24.062 INFO:tasks.rados.rados.0.smithi110.stdout:284: snap_create 2024-09-14T03:38:24.073 INFO:tasks.rados.rados.0.smithi110.stdout:283: expect (ObjNum 97 snap 20 seq_num 97) 2024-09-14T03:38:24.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:24 smithi110 ceph-mon[27365]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:38:24.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:24 smithi110 ceph-mon[35374]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:38:24.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:24 smithi195 ceph-mon[30695]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:38:25.015 INFO:tasks.rados.rados.0.smithi110.stdout:283: done (1 left) 2024-09-14T03:38:25.015 INFO:tasks.rados.rados.0.smithi110.stdout:284: done (0 left) 2024-09-14T03:38:25.015 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 25 2024-09-14T03:38:25.015 INFO:tasks.rados.rados.0.smithi110.stdout:285: seq_num 109 ranges {606208=630784,1236992=491520,1728512=483328,2211840=229376} 2024-09-14T03:38:25.022 INFO:tasks.rados.rados.0.smithi110.stdout:285: writing smithi11075562-23 from 606208 to 1236992 tid 1 2024-09-14T03:38:25.024 INFO:tasks.rados.rados.0.smithi110.stdout:285: writing smithi11075562-23 from 1236992 to 1728512 tid 2 2024-09-14T03:38:25.028 INFO:tasks.rados.rados.0.smithi110.stdout:285: writing smithi11075562-23 from 1728512 to 2211840 tid 3 2024-09-14T03:38:25.029 INFO:tasks.rados.rados.0.smithi110.stdout:285: writing smithi11075562-23 from 2211840 to 2441216 tid 4 2024-09-14T03:38:25.029 INFO:tasks.rados.rados.0.smithi110.stdout:286: rollback oid 9 current snap is 25 2024-09-14T03:38:25.029 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 9 to 22 2024-09-14T03:38:25.030 INFO:tasks.rados.rados.0.smithi110.stdout:285: finishing write tid 1 to smithi11075562-23 2024-09-14T03:38:25.030 INFO:tasks.rados.rados.0.smithi110.stdout:285: finishing write tid 2 to smithi11075562-23 2024-09-14T03:38:25.030 INFO:tasks.rados.rados.0.smithi110.stdout:287: snap_remove snap 17 2024-09-14T03:38:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:25 smithi110 ceph-mon[27365]: pgmap v165: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 5.7 MiB/s wr, 39 op/s 2024-09-14T03:38:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:25 smithi110 ceph-mon[27365]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:38:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:25 smithi110 ceph-mon[27365]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:38:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:25 smithi110 ceph-mon[35374]: pgmap v165: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 5.7 MiB/s wr, 39 op/s 2024-09-14T03:38:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:25 smithi110 ceph-mon[35374]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:38:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:25 smithi110 ceph-mon[35374]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:38:25.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:25 smithi195 ceph-mon[30695]: pgmap v165: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 5.7 MiB/s wr, 39 op/s 2024-09-14T03:38:25.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:25 smithi195 ceph-mon[30695]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:38:25.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:25 smithi195 ceph-mon[30695]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:38:26.017 INFO:tasks.rados.rados.0.smithi110.stdout:285: finishing write tid 3 to smithi11075562-23 2024-09-14T03:38:26.017 INFO:tasks.rados.rados.0.smithi110.stdout:285: finishing write tid 4 to smithi11075562-23 2024-09-14T03:38:26.017 INFO:tasks.rados.rados.0.smithi110.stdout:285: finishing write tid 5 to smithi11075562-23 2024-09-14T03:38:26.017 INFO:tasks.rados.rados.0.smithi110.stdout:285: finishing write tid 6 to smithi11075562-23 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 53 (ObjNum 109 snap 25 seq_num 109) dirty exists 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:285: left oid 23 (ObjNum 109 snap 25 seq_num 109) 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:286: finishing rollback tid 0 to smithi11075562-9 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:286: finishing rollback tid 1 to smithi11075562-9 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:286: finishing rollback tid 2 to smithi11075562-9 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 91 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:285: done (2 left) 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:286: done (1 left) 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:287: done (0 left) 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:288: read oid 7 snap 16 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:288: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:38:26.018 INFO:tasks.rados.rados.0.smithi110.stdout:289: snap_create 2024-09-14T03:38:26.022 INFO:tasks.rados.rados.0.smithi110.stdout:288: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:38:27.017 INFO:tasks.rados.rados.0.smithi110.stdout:288: done (1 left) 2024-09-14T03:38:27.018 INFO:tasks.rados.rados.0.smithi110.stdout:289: done (0 left) 2024-09-14T03:38:27.018 INFO:tasks.rados.rados.0.smithi110.stdout:290: read oid 23 snap -1 2024-09-14T03:38:27.018 INFO:tasks.rados.rados.0.smithi110.stdout:290: expect (ObjNum 109 snap 25 seq_num 109) 2024-09-14T03:38:27.018 INFO:tasks.rados.rados.0.smithi110.stdout:291: snap_remove snap 23 2024-09-14T03:38:27.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:27 smithi110 ceph-mon[27365]: pgmap v168: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:27.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:27 smithi110 ceph-mon[27365]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:38:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:27 smithi110 ceph-mon[35374]: pgmap v168: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:27 smithi110 ceph-mon[35374]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:38:27.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:27 smithi195 ceph-mon[30695]: pgmap v168: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:27.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:27 smithi195 ceph-mon[30695]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:38:28.027 INFO:tasks.rados.rados.0.smithi110.stdout:291: done (1 left) 2024-09-14T03:38:28.027 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 26 2024-09-14T03:38:28.027 INFO:tasks.rados.rados.0.smithi110.stdout:292: seq_num 110 ranges {1802240=647168,2449408=778240,3227648=425984} 2024-09-14T03:38:28.040 INFO:tasks.rados.rados.0.smithi110.stdout:292: writing smithi11075562-45 from 1802240 to 2449408 tid 1 2024-09-14T03:38:28.044 INFO:tasks.rados.rados.0.smithi110.stdout:292: writing smithi11075562-45 from 2449408 to 3227648 tid 2 2024-09-14T03:38:28.047 INFO:tasks.rados.rados.0.smithi110.stdout:292: writing smithi11075562-45 from 3227648 to 3653632 tid 3 2024-09-14T03:38:28.047 INFO:tasks.rados.rados.0.smithi110.stdout:293: rollback oid 35 current snap is 26 2024-09-14T03:38:28.047 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 24 2024-09-14T03:38:28.047 INFO:tasks.rados.rados.0.smithi110.stdout:290: expect (ObjNum 109 snap 25 seq_num 109) 2024-09-14T03:38:28.121 INFO:tasks.rados.rados.0.smithi110.stdout:292: finishing write tid 1 to smithi11075562-45 2024-09-14T03:38:28.121 INFO:tasks.rados.rados.0.smithi110.stdout:293: finishing rollback tid 0 to smithi11075562-35 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:293: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:293: finishing rollback tid 2 to smithi11075562-35 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 81 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:292: finishing write tid 2 to smithi11075562-45 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:290: done (2 left) 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:293: done (1 left) 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:294: snap_create 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:292: finishing write tid 3 to smithi11075562-45 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:292: finishing write tid 4 to smithi11075562-45 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:292: finishing write tid 5 to smithi11075562-45 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 140 (ObjNum 110 snap 26 seq_num 110) dirty exists 2024-09-14T03:38:28.122 INFO:tasks.rados.rados.0.smithi110.stdout:292: left oid 45 (ObjNum 110 snap 26 seq_num 110) 2024-09-14T03:38:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:28 smithi110 ceph-mon[35374]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:38:28.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:28 smithi110 ceph-mon[27365]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:38:28.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:28 smithi195 ceph-mon[30695]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:38:29.027 INFO:tasks.rados.rados.0.smithi110.stdout:292: done (1 left) 2024-09-14T03:38:29.027 INFO:tasks.rados.rados.0.smithi110.stdout:294: done (0 left) 2024-09-14T03:38:29.027 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 27 2024-09-14T03:38:29.027 INFO:tasks.rados.rados.0.smithi110.stdout:295: seq_num 111 ranges {1556480=663552,2220032=663552,2883584=450560,3334144=90112} 2024-09-14T03:38:29.042 INFO:tasks.rados.rados.0.smithi110.stdout:295: writing smithi11075562-18 from 1556480 to 2220032 tid 1 2024-09-14T03:38:29.043 INFO:tasks.rados.rados.0.smithi110.stdout:295: writing smithi11075562-18 from 2220032 to 2883584 tid 2 2024-09-14T03:38:29.045 INFO:tasks.rados.rados.0.smithi110.stdout:295: writing smithi11075562-18 from 2883584 to 3334144 tid 3 2024-09-14T03:38:29.046 INFO:tasks.rados.rados.0.smithi110.stdout:295: writing smithi11075562-18 from 3334144 to 3424256 tid 4 2024-09-14T03:38:29.046 INFO:tasks.rados.rados.0.smithi110.stdout:296: read oid 31 snap -1 2024-09-14T03:38:29.046 INFO:tasks.rados.rados.0.smithi110.stdout:296: expect deleted 2024-09-14T03:38:29.046 INFO:tasks.rados.rados.0.smithi110.stdout:297: read oid 5 snap -1 2024-09-14T03:38:29.046 INFO:tasks.rados.rados.0.smithi110.stdout:297: expect deleted 2024-09-14T03:38:29.046 INFO:tasks.rados.rados.0.smithi110.stdout:298: copy_from oid 21 from oid 32 current snap is 27 2024-09-14T03:38:29.046 INFO:tasks.rados.rados.0.smithi110.stdout:295: finishing write tid 1 to smithi11075562-18 2024-09-14T03:38:29.047 INFO:tasks.rados.rados.0.smithi110.stdout:295: finishing write tid 2 to smithi11075562-18 2024-09-14T03:38:29.047 INFO:tasks.rados.rados.0.smithi110.stdout:299: snap_create 2024-09-14T03:38:29.048 INFO:tasks.rados.rados.0.smithi110.stdout:295: finishing write tid 3 to smithi11075562-18 2024-09-14T03:38:29.048 INFO:tasks.rados.rados.0.smithi110.stdout:295: finishing write tid 4 to smithi11075562-18 2024-09-14T03:38:29.049 INFO:tasks.rados.rados.0.smithi110.stdout:295: finishing write tid 5 to smithi11075562-18 2024-09-14T03:38:29.050 INFO:tasks.rados.rados.0.smithi110.stdout:295: finishing write tid 6 to smithi11075562-18 2024-09-14T03:38:29.051 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 96 (ObjNum 111 snap 27 seq_num 111) dirty exists 2024-09-14T03:38:29.051 INFO:tasks.rados.rados.0.smithi110.stdout:295: left oid 18 (ObjNum 111 snap 27 seq_num 111) 2024-09-14T03:38:29.056 INFO:tasks.rados.rados.0.smithi110.stdout:298: finishing copy_from racing read to smithi11075562-21 2024-09-14T03:38:29.058 INFO:tasks.rados.rados.0.smithi110.stdout:298: finishing copy_from to smithi11075562-21 2024-09-14T03:38:29.058 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 58 (ObjNum 100 snap 20 seq_num 100) dirty exists 2024-09-14T03:38:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:29 smithi110 ceph-mon[27365]: pgmap v171: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:38:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:29 smithi110 ceph-mon[27365]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:38:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:29 smithi110 ceph-mon[35374]: pgmap v171: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:38:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:29 smithi110 ceph-mon[35374]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:38:29.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:29 smithi195 ceph-mon[30695]: pgmap v171: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:38:29.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:29 smithi195 ceph-mon[30695]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:38:30.016 INFO:tasks.rados.rados.0.smithi110.stdout:295: done (4 left) 2024-09-14T03:38:30.016 INFO:tasks.rados.rados.0.smithi110.stdout:296: done (3 left) 2024-09-14T03:38:30.017 INFO:tasks.rados.rados.0.smithi110.stdout:297: done (2 left) 2024-09-14T03:38:30.017 INFO:tasks.rados.rados.0.smithi110.stdout:298: done (1 left) 2024-09-14T03:38:30.017 INFO:tasks.rados.rados.0.smithi110.stdout:299: done (0 left) 2024-09-14T03:38:30.017 INFO:tasks.rados.rados.0.smithi110.stdout:300: read oid 41 snap -1 2024-09-14T03:38:30.017 INFO:tasks.rados.rados.0.smithi110.stdout:300: expect deleted 2024-09-14T03:38:30.017 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 28 2024-09-14T03:38:30.017 INFO:tasks.rados.rados.0.smithi110.stdout:301: seq_num 112 ranges {3203072=679936,3883008=548864,4431872=655360} 2024-09-14T03:38:30.037 INFO:tasks.rados.rados.0.smithi110.stdout:301: writing smithi11075562-16 from 3203072 to 3883008 tid 1 2024-09-14T03:38:30.040 INFO:tasks.rados.rados.0.smithi110.stdout:301: writing smithi11075562-16 from 3883008 to 4431872 tid 2 2024-09-14T03:38:30.044 INFO:tasks.rados.rados.0.smithi110.stdout:301: writing smithi11075562-16 from 4431872 to 5087232 tid 3 2024-09-14T03:38:30.044 INFO:tasks.rados.rados.0.smithi110.stdout:302: setattr oid 13 current snap is 28 2024-09-14T03:38:30.044 INFO:tasks.rados.rados.0.smithi110.stdout:301: finishing write tid 1 to smithi11075562-16 2024-09-14T03:38:30.044 INFO:tasks.rados.rados.0.smithi110.stdout:301: finishing write tid 2 to smithi11075562-16 2024-09-14T03:38:30.045 INFO:tasks.rados.rados.0.smithi110.stdout:300: done (2 left) 2024-09-14T03:38:30.045 INFO:tasks.rados.rados.0.smithi110.stdout:303: delete oid 37 current snap is 28 2024-09-14T03:38:30.047 INFO:tasks.rados.rados.0.smithi110.stdout:301: finishing write tid 3 to smithi11075562-16 2024-09-14T03:38:30.047 INFO:tasks.rados.rados.0.smithi110.stdout:301: finishing write tid 4 to smithi11075562-16 2024-09-14T03:38:30.049 INFO:tasks.rados.rados.0.smithi110.stdout:301: finishing write tid 5 to smithi11075562-16 2024-09-14T03:38:30.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 101 (ObjNum 112 snap 28 seq_num 112) dirty exists 2024-09-14T03:38:30.049 INFO:tasks.rados.rados.0.smithi110.stdout:301: left oid 16 (ObjNum 112 snap 28 seq_num 112) 2024-09-14T03:38:30.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 83 (ObjNum 80 snap 13 seq_num 80) dirty exists 2024-09-14T03:38:30.052 INFO:tasks.rados.rados.0.smithi110.stdout:301: done (2 left) 2024-09-14T03:38:30.052 INFO:tasks.rados.rados.0.smithi110.stdout:302: done (1 left) 2024-09-14T03:38:30.052 INFO:tasks.rados.rados.0.smithi110.stdout:303: done (0 left) 2024-09-14T03:38:30.052 INFO:tasks.rados.rados.0.smithi110.stdout:304: read oid 35 snap -1 2024-09-14T03:38:30.052 INFO:tasks.rados.rados.0.smithi110.stdout:304: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:38:30.052 INFO:tasks.rados.rados.0.smithi110.stdout:305: read oid 10 snap 16 2024-09-14T03:38:30.052 INFO:tasks.rados.rados.0.smithi110.stdout:305: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-14T03:38:30.053 INFO:tasks.rados.rados.0.smithi110.stdout:306: delete oid 20 current snap is 28 2024-09-14T03:38:30.054 INFO:tasks.rados.rados.0.smithi110.stdout:304: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:38:30.057 INFO:tasks.rados.rados.0.smithi110.stdout:304: done (2 left) 2024-09-14T03:38:30.057 INFO:tasks.rados.rados.0.smithi110.stdout:306: done (1 left) 2024-09-14T03:38:30.057 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 28 2024-09-14T03:38:30.057 INFO:tasks.rados.rados.0.smithi110.stdout:307: seq_num 115 ranges {1228800=729088,1957888=647168,2605056=434176,3039232=122880} 2024-09-14T03:38:30.067 INFO:tasks.rados.rados.0.smithi110.stdout:307: writing smithi11075562-1 from 1228800 to 1957888 tid 1 2024-09-14T03:38:30.071 INFO:tasks.rados.rados.0.smithi110.stdout:307: writing smithi11075562-1 from 1957888 to 2605056 tid 2 2024-09-14T03:38:30.073 INFO:tasks.rados.rados.0.smithi110.stdout:307: writing smithi11075562-1 from 2605056 to 3039232 tid 3 2024-09-14T03:38:30.074 INFO:tasks.rados.rados.0.smithi110.stdout:307: writing smithi11075562-1 from 3039232 to 3162112 tid 4 2024-09-14T03:38:30.074 INFO:tasks.rados.rados.0.smithi110.stdout:308: delete oid 17 current snap is 28 2024-09-14T03:38:30.074 INFO:tasks.rados.rados.0.smithi110.stdout:305: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:307: finishing write tid 1 to smithi11075562-1 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:307: finishing write tid 2 to smithi11075562-1 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:307: finishing write tid 3 to smithi11075562-1 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:307: finishing write tid 4 to smithi11075562-1 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:307: finishing write tid 5 to smithi11075562-1 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:307: finishing write tid 6 to smithi11075562-1 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 89 (ObjNum 115 snap 28 seq_num 115) dirty exists 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:307: left oid 1 (ObjNum 115 snap 28 seq_num 115) 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:305: done (2 left) 2024-09-14T03:38:30.087 INFO:tasks.rados.rados.0.smithi110.stdout:307: done (1 left) 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:308: done (0 left) 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:309: copy_from oid 32 from oid 23 current snap is 28 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:310: read oid 7 snap -1 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:310: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:311: read oid 19 snap -1 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:311: expect (ObjNum 66 snap 6 seq_num 66) 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:312: copy_from oid 5 from oid 44 current snap is 28 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:313: rmattr oid 36 current snap is 28 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:313: done (4 left) 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:314: read oid 45 snap -1 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:314: expect (ObjNum 110 snap 26 seq_num 110) 2024-09-14T03:38:30.088 INFO:tasks.rados.rados.0.smithi110.stdout:315: setattr oid 29 current snap is 28 2024-09-14T03:38:30.089 INFO:tasks.rados.rados.0.smithi110.stdout:316: rollback oid 31 current snap is 28 2024-09-14T03:38:30.089 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 22 2024-09-14T03:38:30.089 INFO:tasks.rados.rados.0.smithi110.stdout:317: snap_create 2024-09-14T03:38:30.089 INFO:tasks.rados.rados.0.smithi110.stdout:316: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:38:30.089 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:38:30.089 INFO:tasks.rados.rados.0.smithi110.stdout:310: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:38:30.092 INFO:tasks.rados.rados.0.smithi110.stdout:312: finishing copy_from to smithi11075562-5 2024-09-14T03:38:30.092 INFO:tasks.rados.rados.0.smithi110.stdout:312: got expected ENOENT (src dne) 2024-09-14T03:38:30.093 INFO:tasks.rados.rados.0.smithi110.stdout:312: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:38:30.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 48 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:38:30.095 INFO:tasks.rados.rados.0.smithi110.stdout:311: expect (ObjNum 66 snap 6 seq_num 66) 2024-09-14T03:38:30.140 INFO:tasks.rados.rados.0.smithi110.stdout:309: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:38:30.140 INFO:tasks.rados.rados.0.smithi110.stdout:309: finishing copy_from to smithi11075562-32 2024-09-14T03:38:30.140 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 53 (ObjNum 109 snap 25 seq_num 109) dirty exists 2024-09-14T03:38:30.140 INFO:tasks.rados.rados.0.smithi110.stdout:314: expect (ObjNum 110 snap 26 seq_num 110) 2024-09-14T03:38:30.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:30 smithi110 ceph-mon[27365]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:38:30.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:30 smithi110 ceph-mon[27365]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:38:30.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:30 smithi110 ceph-mon[35374]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:38:30.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:30 smithi110 ceph-mon[35374]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:38:30.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:30 smithi195 ceph-mon[30695]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:38:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:30 smithi195 ceph-mon[30695]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:38:30.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[32531]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:38:30.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:30] "GET /metrics HTTP/1.1" 200 239895 "" "Prometheus/2.43.0" 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:309: done (7 left) 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:310: done (6 left) 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:311: done (5 left) 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:312: done (4 left) 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:314: done (3 left) 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:315: done (2 left) 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:316: done (1 left) 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:317: done (0 left) 2024-09-14T03:38:31.040 INFO:tasks.rados.rados.0.smithi110.stdout:318: rollback oid 50 current snap is 29 2024-09-14T03:38:31.041 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 25 2024-09-14T03:38:31.041 INFO:tasks.rados.rados.0.smithi110.stdout:319: rmattr oid 38 current snap is 29 2024-09-14T03:38:31.041 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 29 2024-09-14T03:38:31.041 INFO:tasks.rados.rados.0.smithi110.stdout:320: seq_num 117 ranges {425984=770048,1196032=417792,1613824=442368,2056192=344064} 2024-09-14T03:38:31.047 INFO:tasks.rados.rados.0.smithi110.stdout:320: writing smithi11075562-39 from 425984 to 1196032 tid 1 2024-09-14T03:38:31.049 INFO:tasks.rados.rados.0.smithi110.stdout:320: writing smithi11075562-39 from 1196032 to 1613824 tid 2 2024-09-14T03:38:31.052 INFO:tasks.rados.rados.0.smithi110.stdout:320: writing smithi11075562-39 from 1613824 to 2056192 tid 3 2024-09-14T03:38:31.054 INFO:tasks.rados.rados.0.smithi110.stdout:320: writing smithi11075562-39 from 2056192 to 2400256 tid 4 2024-09-14T03:38:31.054 INFO:tasks.rados.rados.0.smithi110.stdout:321: copy_from oid 17 from oid 9 current snap is 29 2024-09-14T03:38:31.055 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 49 (ObjNum 102 snap 20 seq_num 102) dirty exists 2024-09-14T03:38:31.055 INFO:tasks.rados.rados.0.smithi110.stdout:318: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:38:31.055 INFO:tasks.rados.rados.0.smithi110.stdout:318: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:38:31.055 INFO:tasks.rados.rados.0.smithi110.stdout:318: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:38:31.055 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 7 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:38:31.055 INFO:tasks.rados.rados.0.smithi110.stdout:320: finishing write tid 1 to smithi11075562-39 2024-09-14T03:38:31.055 INFO:tasks.rados.rados.0.smithi110.stdout:320: finishing write tid 2 to smithi11075562-39 2024-09-14T03:38:31.055 INFO:tasks.rados.rados.0.smithi110.stdout:320: finishing write tid 3 to smithi11075562-39 2024-09-14T03:38:31.056 INFO:tasks.rados.rados.0.smithi110.stdout:318: done (3 left) 2024-09-14T03:38:31.056 INFO:tasks.rados.rados.0.smithi110.stdout:319: done (2 left) 2024-09-14T03:38:31.056 INFO:tasks.rados.rados.0.smithi110.stdout:322: rmattr oid 43 current snap is 29 2024-09-14T03:38:31.056 INFO:tasks.rados.rados.0.smithi110.stdout:322: done (2 left) 2024-09-14T03:38:31.056 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 29 2024-09-14T03:38:31.056 INFO:tasks.rados.rados.0.smithi110.stdout:323: seq_num 118 ranges {1802240=786432,2588672=704512,3293184=499712} 2024-09-14T03:38:31.068 INFO:tasks.rados.rados.0.smithi110.stdout:323: writing smithi11075562-14 from 1802240 to 2588672 tid 1 2024-09-14T03:38:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:323: writing smithi11075562-14 from 2588672 to 3293184 tid 2 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:323: writing smithi11075562-14 from 3293184 to 3792896 tid 3 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 29 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:320: finishing write tid 4 to smithi11075562-39 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:320: finishing write tid 5 to smithi11075562-39 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:320: finishing write tid 6 to smithi11075562-39 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 94 (ObjNum 117 snap 29 seq_num 117) dirty exists 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:320: left oid 39 (ObjNum 117 snap 29 seq_num 117) 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:321: finishing copy_from racing read to smithi11075562-17 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:321: finishing copy_from to smithi11075562-17 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 91 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-14T03:38:31.076 INFO:tasks.rados.rados.0.smithi110.stdout:323: finishing write tid 1 to smithi11075562-14 2024-09-14T03:38:31.077 INFO:tasks.rados.rados.0.smithi110.stdout:324: seq_num 119 ranges {1794048=802816,2596864=589824,3186688=409600,3596288=204800} 2024-09-14T03:38:31.090 INFO:tasks.rados.rados.0.smithi110.stdout:324: writing smithi11075562-21 from 1794048 to 2596864 tid 1 2024-09-14T03:38:31.093 INFO:tasks.rados.rados.0.smithi110.stdout:324: writing smithi11075562-21 from 2596864 to 3186688 tid 2 2024-09-14T03:38:31.095 INFO:tasks.rados.rados.0.smithi110.stdout:324: writing smithi11075562-21 from 3186688 to 3596288 tid 3 2024-09-14T03:38:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:324: writing smithi11075562-21 from 3596288 to 3801088 tid 4 2024-09-14T03:38:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:320: done (3 left) 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:321: done (2 left) 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:325: copy_from oid 27 from oid 35 current snap is 29 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:323: finishing write tid 2 to smithi11075562-14 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:323: finishing write tid 3 to smithi11075562-14 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:323: finishing write tid 4 to smithi11075562-14 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:323: finishing write tid 5 to smithi11075562-14 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 99 (ObjNum 118 snap 29 seq_num 118) dirty exists 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:323: left oid 14 (ObjNum 118 snap 29 seq_num 118) 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:324: finishing write tid 1 to smithi11075562-21 2024-09-14T03:38:31.098 INFO:tasks.rados.rados.0.smithi110.stdout:324: finishing write tid 2 to smithi11075562-21 2024-09-14T03:38:31.099 INFO:tasks.rados.rados.0.smithi110.stdout:323: done (2 left) 2024-09-14T03:38:31.099 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 29 2024-09-14T03:38:31.099 INFO:tasks.rados.rados.0.smithi110.stdout:326: seq_num 120 ranges {1703936=417792,2121728=475136,2596864=622592,3219456=507904} 2024-09-14T03:38:31.108 INFO:tasks.rados.rados.0.smithi110.stdout:326: writing smithi11075562-11 from 1703936 to 2121728 tid 1 2024-09-14T03:38:31.111 INFO:tasks.rados.rados.0.smithi110.stdout:326: writing smithi11075562-11 from 2121728 to 2596864 tid 2 2024-09-14T03:38:31.114 INFO:tasks.rados.rados.0.smithi110.stdout:326: writing smithi11075562-11 from 2596864 to 3219456 tid 3 2024-09-14T03:38:31.117 INFO:tasks.rados.rados.0.smithi110.stdout:326: writing smithi11075562-11 from 3219456 to 3727360 tid 4 2024-09-14T03:38:31.117 INFO:tasks.rados.rados.0.smithi110.stdout:327: snap_remove snap 26 2024-09-14T03:38:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:31 smithi110 ceph-mon[27365]: pgmap v174: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:38:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:31 smithi110 ceph-mon[35374]: pgmap v174: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:38:31.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:31 smithi195 ceph-mon[30695]: pgmap v174: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:38:32.045 INFO:tasks.rados.rados.0.smithi110.stdout:327: done (3 left) 2024-09-14T03:38:32.045 INFO:tasks.rados.rados.0.smithi110.stdout:328: copy_from oid 16 from oid 44 current snap is 29 2024-09-14T03:38:32.045 INFO:tasks.rados.rados.0.smithi110.stdout:329: copy_from oid 2 from oid 46 current snap is 29 2024-09-14T03:38:32.045 INFO:tasks.rados.rados.0.smithi110.stdout:324: finishing write tid 3 to smithi11075562-21 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:324: finishing write tid 4 to smithi11075562-21 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:324: finishing write tid 5 to smithi11075562-21 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:324: finishing write tid 6 to smithi11075562-21 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 105 (ObjNum 119 snap 29 seq_num 119) dirty exists 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:324: left oid 21 (ObjNum 119 snap 29 seq_num 119) 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:325: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:325: finishing copy_from to smithi11075562-27 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 81 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:326: finishing write tid 1 to smithi11075562-11 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:326: finishing write tid 2 to smithi11075562-11 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:326: finishing write tid 3 to smithi11075562-11 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:326: finishing write tid 4 to smithi11075562-11 2024-09-14T03:38:32.046 INFO:tasks.rados.rados.0.smithi110.stdout:326: finishing write tid 5 to smithi11075562-11 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:326: finishing write tid 6 to smithi11075562-11 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 96 (ObjNum 120 snap 29 seq_num 120) dirty exists 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:326: left oid 11 (ObjNum 120 snap 29 seq_num 120) 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:324: done (4 left) 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:325: done (3 left) 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:326: done (2 left) 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:330: snap_create 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:328: finishing copy_from to smithi11075562-16 2024-09-14T03:38:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:328: got expected ENOENT (src dne) 2024-09-14T03:38:32.049 INFO:tasks.rados.rados.0.smithi110.stdout:328: finishing copy_from racing read to smithi11075562-16 2024-09-14T03:38:32.056 INFO:tasks.rados.rados.0.smithi110.stdout:329: finishing copy_from racing read to smithi11075562-2 2024-09-14T03:38:32.057 INFO:tasks.rados.rados.0.smithi110.stdout:329: finishing copy_from to smithi11075562-2 2024-09-14T03:38:32.057 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 42 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:38:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:32 smithi110 ceph-mon[27365]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:38:32.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:32 smithi110 ceph-mon[35374]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:38:32.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:32 smithi195 ceph-mon[30695]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:38:33.050 INFO:tasks.rados.rados.0.smithi110.stdout:328: done (2 left) 2024-09-14T03:38:33.050 INFO:tasks.rados.rados.0.smithi110.stdout:329: done (1 left) 2024-09-14T03:38:33.050 INFO:tasks.rados.rados.0.smithi110.stdout:330: done (0 left) 2024-09-14T03:38:33.050 INFO:tasks.rados.rados.0.smithi110.stdout:331: read oid 15 snap -1 2024-09-14T03:38:33.050 INFO:tasks.rados.rados.0.smithi110.stdout:331: expect (ObjNum 97 snap 20 seq_num 97) 2024-09-14T03:38:33.050 INFO:tasks.rados.rados.0.smithi110.stdout:332: copy_from oid 39 from oid 19 current snap is 30 2024-09-14T03:38:33.050 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 30 2024-09-14T03:38:33.050 INFO:tasks.rados.rados.0.smithi110.stdout:333: seq_num 121 ranges {1384448=434176,1818624=753664,2572288=425984,2998272=425984} 2024-09-14T03:38:33.060 INFO:tasks.rados.rados.0.smithi110.stdout:333: writing smithi11075562-9 from 1384448 to 1818624 tid 1 2024-09-14T03:38:33.065 INFO:tasks.rados.rados.0.smithi110.stdout:333: writing smithi11075562-9 from 1818624 to 2572288 tid 2 2024-09-14T03:38:33.067 INFO:tasks.rados.rados.0.smithi110.stdout:333: writing smithi11075562-9 from 2572288 to 2998272 tid 3 2024-09-14T03:38:33.070 INFO:tasks.rados.rados.0.smithi110.stdout:333: writing smithi11075562-9 from 2998272 to 3424256 tid 4 2024-09-14T03:38:33.070 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 30 2024-09-14T03:38:33.070 INFO:tasks.rados.rados.0.smithi110.stdout:334: seq_num 122 ranges {3653632=450560,4104192=688128,4792320=630784,5423104=286720} 2024-09-14T03:38:33.091 INFO:tasks.rados.rados.0.smithi110.stdout:334: writing smithi11075562-45 from 3653632 to 4104192 tid 1 2024-09-14T03:38:33.095 INFO:tasks.rados.rados.0.smithi110.stdout:334: writing smithi11075562-45 from 4104192 to 4792320 tid 2 2024-09-14T03:38:33.098 INFO:tasks.rados.rados.0.smithi110.stdout:334: writing smithi11075562-45 from 4792320 to 5423104 tid 3 2024-09-14T03:38:33.100 INFO:tasks.rados.rados.0.smithi110.stdout:334: writing smithi11075562-45 from 5423104 to 5709824 tid 4 2024-09-14T03:38:33.101 INFO:tasks.rados.rados.0.smithi110.stdout:335: snap_create 2024-09-14T03:38:33.101 INFO:tasks.rados.rados.0.smithi110.stdout:332: finishing copy_from racing read to smithi11075562-39 2024-09-14T03:38:33.101 INFO:tasks.rados.rados.0.smithi110.stdout:331: expect (ObjNum 97 snap 20 seq_num 97) 2024-09-14T03:38:33.187 INFO:tasks.rados.rados.0.smithi110.stdout:332: finishing copy_from to smithi11075562-39 2024-09-14T03:38:33.187 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 44 (ObjNum 66 snap 6 seq_num 66) dirty exists 2024-09-14T03:38:33.187 INFO:tasks.rados.rados.0.smithi110.stdout:333: finishing write tid 1 to smithi11075562-9 2024-09-14T03:38:33.187 INFO:tasks.rados.rados.0.smithi110.stdout:333: finishing write tid 2 to smithi11075562-9 2024-09-14T03:38:33.188 INFO:tasks.rados.rados.0.smithi110.stdout:333: finishing write tid 3 to smithi11075562-9 2024-09-14T03:38:33.188 INFO:tasks.rados.rados.0.smithi110.stdout:333: finishing write tid 4 to smithi11075562-9 2024-09-14T03:38:33.188 INFO:tasks.rados.rados.0.smithi110.stdout:333: finishing write tid 5 to smithi11075562-9 2024-09-14T03:38:33.188 INFO:tasks.rados.rados.0.smithi110.stdout:333: finishing write tid 6 to smithi11075562-9 2024-09-14T03:38:33.188 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 109 (ObjNum 121 snap 30 seq_num 121) dirty exists 2024-09-14T03:38:33.188 INFO:tasks.rados.rados.0.smithi110.stdout:333: left oid 9 (ObjNum 121 snap 30 seq_num 121) 2024-09-14T03:38:33.188 INFO:tasks.rados.rados.0.smithi110.stdout:334: finishing write tid 1 to smithi11075562-45 2024-09-14T03:38:33.188 INFO:tasks.rados.rados.0.smithi110.stdout:334: finishing write tid 2 to smithi11075562-45 2024-09-14T03:38:33.189 INFO:tasks.rados.rados.0.smithi110.stdout:334: finishing write tid 3 to smithi11075562-45 2024-09-14T03:38:33.189 INFO:tasks.rados.rados.0.smithi110.stdout:334: finishing write tid 4 to smithi11075562-45 2024-09-14T03:38:33.189 INFO:tasks.rados.rados.0.smithi110.stdout:334: finishing write tid 5 to smithi11075562-45 2024-09-14T03:38:33.189 INFO:tasks.rados.rados.0.smithi110.stdout:334: finishing write tid 6 to smithi11075562-45 2024-09-14T03:38:33.189 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 150 (ObjNum 122 snap 30 seq_num 122) dirty exists 2024-09-14T03:38:33.189 INFO:tasks.rados.rados.0.smithi110.stdout:334: left oid 45 (ObjNum 122 snap 30 seq_num 122) 2024-09-14T03:38:33.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:33 smithi110 ceph-mon[35374]: pgmap v177: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T03:38:33.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:33 smithi110 ceph-mon[35374]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:38:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:33 smithi110 ceph-mon[27365]: pgmap v177: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T03:38:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:33 smithi110 ceph-mon[27365]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:38:33.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:33 smithi195 ceph-mon[30695]: pgmap v177: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T03:38:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:33 smithi195 ceph-mon[30695]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:38:34.054 INFO:tasks.rados.rados.0.smithi110.stdout:331: done (4 left) 2024-09-14T03:38:34.054 INFO:tasks.rados.rados.0.smithi110.stdout:332: done (3 left) 2024-09-14T03:38:34.054 INFO:tasks.rados.rados.0.smithi110.stdout:333: done (2 left) 2024-09-14T03:38:34.054 INFO:tasks.rados.rados.0.smithi110.stdout:334: done (1 left) 2024-09-14T03:38:34.054 INFO:tasks.rados.rados.0.smithi110.stdout:335: done (0 left) 2024-09-14T03:38:34.054 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 31 2024-09-14T03:38:34.054 INFO:tasks.rados.rados.0.smithi110.stdout:336: seq_num 123 ranges {0=466944,466944=573440,1040384=794624,1835008=237568} 2024-09-14T03:38:34.056 INFO:tasks.rados.rados.0.smithi110.stdout:336: writing smithi11075562-5 from 0 to 466944 tid 1 2024-09-14T03:38:34.059 INFO:tasks.rados.rados.0.smithi110.stdout:336: writing smithi11075562-5 from 466944 to 1040384 tid 2 2024-09-14T03:38:34.064 INFO:tasks.rados.rados.0.smithi110.stdout:336: writing smithi11075562-5 from 1040384 to 1835008 tid 3 2024-09-14T03:38:34.066 INFO:tasks.rados.rados.0.smithi110.stdout:336: writing smithi11075562-5 from 1835008 to 2072576 tid 4 2024-09-14T03:38:34.066 INFO:tasks.rados.rados.0.smithi110.stdout:337: rollback oid 31 current snap is 31 2024-09-14T03:38:34.066 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 24 2024-09-14T03:38:34.066 INFO:tasks.rados.rados.0.smithi110.stdout:336: finishing write tid 1 to smithi11075562-5 2024-09-14T03:38:34.066 INFO:tasks.rados.rados.0.smithi110.stdout:336: finishing write tid 2 to smithi11075562-5 2024-09-14T03:38:34.066 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 31 2024-09-14T03:38:34.066 INFO:tasks.rados.rados.0.smithi110.stdout:338: seq_num 124 ranges {3932160=483328,4415488=458752,4874240=598016,5472256=548864} 2024-09-14T03:38:34.088 INFO:tasks.rados.rados.0.smithi110.stdout:338: writing smithi11075562-49 from 3932160 to 4415488 tid 1 2024-09-14T03:38:34.091 INFO:tasks.rados.rados.0.smithi110.stdout:338: writing smithi11075562-49 from 4415488 to 4874240 tid 2 2024-09-14T03:38:34.094 INFO:tasks.rados.rados.0.smithi110.stdout:338: writing smithi11075562-49 from 4874240 to 5472256 tid 3 2024-09-14T03:38:34.098 INFO:tasks.rados.rados.0.smithi110.stdout:338: writing smithi11075562-49 from 5472256 to 6021120 tid 4 2024-09-14T03:38:34.098 INFO:tasks.rados.rados.0.smithi110.stdout:339: rmattr oid 41 current snap is 31 2024-09-14T03:38:34.098 INFO:tasks.rados.rados.0.smithi110.stdout:339: done (3 left) 2024-09-14T03:38:34.098 INFO:tasks.rados.rados.0.smithi110.stdout:340: read oid 7 snap -1 2024-09-14T03:38:34.098 INFO:tasks.rados.rados.0.smithi110.stdout:340: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:38:34.098 INFO:tasks.rados.rados.0.smithi110.stdout:341: snap_create 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:337: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 0 (ObjNum 33 snap 0 seq_num 43181432) dirty dne 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:336: finishing write tid 3 to smithi11075562-5 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:336: finishing write tid 4 to smithi11075562-5 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:336: finishing write tid 5 to smithi11075562-5 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:336: finishing write tid 6 to smithi11075562-5 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 155 (ObjNum 123 snap 31 seq_num 123) dirty exists 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:336: left oid 5 (ObjNum 123 snap 31 seq_num 123) 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:338: finishing write tid 1 to smithi11075562-49 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:338: finishing write tid 2 to smithi11075562-49 2024-09-14T03:38:34.099 INFO:tasks.rados.rados.0.smithi110.stdout:338: finishing write tid 3 to smithi11075562-49 2024-09-14T03:38:34.100 INFO:tasks.rados.rados.0.smithi110.stdout:340: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:38:34.103 INFO:tasks.rados.rados.0.smithi110.stdout:338: finishing write tid 4 to smithi11075562-49 2024-09-14T03:38:34.103 INFO:tasks.rados.rados.0.smithi110.stdout:338: finishing write tid 5 to smithi11075562-49 2024-09-14T03:38:34.103 INFO:tasks.rados.rados.0.smithi110.stdout:338: finishing write tid 6 to smithi11075562-49 2024-09-14T03:38:34.103 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 99 (ObjNum 124 snap 31 seq_num 124) dirty exists 2024-09-14T03:38:34.103 INFO:tasks.rados.rados.0.smithi110.stdout:338: left oid 49 (ObjNum 124 snap 31 seq_num 124) 2024-09-14T03:38:34.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:34 smithi110 ceph-mon[35374]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:38:34.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:34 smithi110 ceph-mon[27365]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:38:34.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:34 smithi195 ceph-mon[30695]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:38:35.019 INFO:tasks.rados.rados.0.smithi110.stdout:336: done (4 left) 2024-09-14T03:38:35.019 INFO:tasks.rados.rados.0.smithi110.stdout:337: done (3 left) 2024-09-14T03:38:35.019 INFO:tasks.rados.rados.0.smithi110.stdout:338: done (2 left) 2024-09-14T03:38:35.019 INFO:tasks.rados.rados.0.smithi110.stdout:340: done (1 left) 2024-09-14T03:38:35.020 INFO:tasks.rados.rados.0.smithi110.stdout:341: done (0 left) 2024-09-14T03:38:35.020 INFO:tasks.rados.rados.0.smithi110.stdout:342: delete oid 49 current snap is 32 2024-09-14T03:38:35.021 INFO:tasks.rados.rados.0.smithi110.stdout:342: done (0 left) 2024-09-14T03:38:35.022 INFO:tasks.rados.rados.0.smithi110.stdout:343: rollback oid 18 current snap is 32 2024-09-14T03:38:35.022 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 18 to 27 2024-09-14T03:38:35.022 INFO:tasks.rados.rados.0.smithi110.stdout:344: snap_create 2024-09-14T03:38:35.025 INFO:tasks.rados.rados.0.smithi110.stdout:343: finishing rollback tid 0 to smithi11075562-18 2024-09-14T03:38:35.026 INFO:tasks.rados.rados.0.smithi110.stdout:343: finishing rollback tid 1 to smithi11075562-18 2024-09-14T03:38:35.026 INFO:tasks.rados.rados.0.smithi110.stdout:343: finishing rollback tid 2 to smithi11075562-18 2024-09-14T03:38:35.026 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 109 (ObjNum 111 snap 27 seq_num 111) dirty exists 2024-09-14T03:38:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:35 smithi110 ceph-mon[35374]: pgmap v180: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T03:38:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:35 smithi110 ceph-mon[35374]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:38:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:35 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:35 smithi110 ceph-mon[35374]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:38:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:35 smithi110 ceph-mon[27365]: pgmap v180: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T03:38:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:35 smithi110 ceph-mon[27365]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:38:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:35 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:35 smithi110 ceph-mon[27365]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:38:35.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:35 smithi195 ceph-mon[30695]: pgmap v180: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T03:38:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:35 smithi195 ceph-mon[30695]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:38:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:35 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:35 smithi195 ceph-mon[30695]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:38:36.026 INFO:tasks.rados.rados.0.smithi110.stdout:343: done (1 left) 2024-09-14T03:38:36.026 INFO:tasks.rados.rados.0.smithi110.stdout:344: done (0 left) 2024-09-14T03:38:36.026 INFO:tasks.rados.rados.0.smithi110.stdout:345: rollback oid 41 current snap is 33 2024-09-14T03:38:36.026 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 24 2024-09-14T03:38:36.026 INFO:tasks.rados.rados.0.smithi110.stdout:346: rollback oid 18 current snap is 33 2024-09-14T03:38:36.027 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 18 to 25 2024-09-14T03:38:36.027 INFO:tasks.rados.rados.0.smithi110.stdout:347: rmattr oid 23 current snap is 33 2024-09-14T03:38:36.027 INFO:tasks.rados.rados.0.smithi110.stdout:347: done (2 left) 2024-09-14T03:38:36.027 INFO:tasks.rados.rados.0.smithi110.stdout:348: copy_from oid 43 from oid 10 current snap is 33 2024-09-14T03:38:36.027 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 33 2024-09-14T03:38:36.027 INFO:tasks.rados.rados.0.smithi110.stdout:349: seq_num 126 ranges {1417216=516096,1933312=630784,2564096=614400,3178496=360448} 2024-09-14T03:38:36.036 INFO:tasks.rados.rados.0.smithi110.stdout:349: writing smithi11075562-19 from 1417216 to 1933312 tid 1 2024-09-14T03:38:36.040 INFO:tasks.rados.rados.0.smithi110.stdout:349: writing smithi11075562-19 from 1933312 to 2564096 tid 2 2024-09-14T03:38:36.043 INFO:tasks.rados.rados.0.smithi110.stdout:349: writing smithi11075562-19 from 2564096 to 3178496 tid 3 2024-09-14T03:38:36.046 INFO:tasks.rados.rados.0.smithi110.stdout:349: writing smithi11075562-19 from 3178496 to 3538944 tid 4 2024-09-14T03:38:36.046 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 33 2024-09-14T03:38:36.046 INFO:tasks.rados.rados.0.smithi110.stdout:345: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:38:36.046 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 0 (ObjNum 305 snap 0 seq_num 1) dirty dne 2024-09-14T03:38:36.046 INFO:tasks.rados.rados.0.smithi110.stdout:350: seq_num 127 ranges {1556480=532480,2088960=507904,2596864=778240,3375104=319488} 2024-09-14T03:38:36.057 INFO:tasks.rados.rados.0.smithi110.stdout:350: writing smithi11075562-13 from 1556480 to 2088960 tid 1 2024-09-14T03:38:36.060 INFO:tasks.rados.rados.0.smithi110.stdout:350: writing smithi11075562-13 from 2088960 to 2596864 tid 2 2024-09-14T03:38:36.064 INFO:tasks.rados.rados.0.smithi110.stdout:350: writing smithi11075562-13 from 2596864 to 3375104 tid 3 2024-09-14T03:38:36.066 INFO:tasks.rados.rados.0.smithi110.stdout:350: writing smithi11075562-13 from 3375104 to 3694592 tid 4 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:345: done (4 left) 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:351: setattr oid 41 current snap is 33 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:346: finishing rollback tid 0 to smithi11075562-18 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:346: finishing rollback tid 1 to smithi11075562-18 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:346: finishing rollback tid 2 to smithi11075562-18 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 113 (ObjNum 80 snap 13 seq_num 80) dirty exists 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:348: finishing copy_from racing read to smithi11075562-43 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:348: finishing copy_from to smithi11075562-43 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 2 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:349: finishing write tid 1 to smithi11075562-19 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:349: finishing write tid 2 to smithi11075562-19 2024-09-14T03:38:36.067 INFO:tasks.rados.rados.0.smithi110.stdout:349: finishing write tid 3 to smithi11075562-19 2024-09-14T03:38:36.068 INFO:tasks.rados.rados.0.smithi110.stdout:349: finishing write tid 4 to smithi11075562-19 2024-09-14T03:38:36.068 INFO:tasks.rados.rados.0.smithi110.stdout:349: finishing write tid 5 to smithi11075562-19 2024-09-14T03:38:36.068 INFO:tasks.rados.rados.0.smithi110.stdout:349: finishing write tid 6 to smithi11075562-19 2024-09-14T03:38:36.068 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 119 (ObjNum 126 snap 33 seq_num 126) dirty exists 2024-09-14T03:38:36.068 INFO:tasks.rados.rados.0.smithi110.stdout:349: left oid 19 (ObjNum 126 snap 33 seq_num 126) 2024-09-14T03:38:36.068 INFO:tasks.rados.rados.0.smithi110.stdout:350: finishing write tid 1 to smithi11075562-13 2024-09-14T03:38:36.068 INFO:tasks.rados.rados.0.smithi110.stdout:350: finishing write tid 2 to smithi11075562-13 2024-09-14T03:38:36.069 INFO:tasks.rados.rados.0.smithi110.stdout:350: finishing write tid 3 to smithi11075562-13 2024-09-14T03:38:36.069 INFO:tasks.rados.rados.0.smithi110.stdout:350: finishing write tid 4 to smithi11075562-13 2024-09-14T03:38:36.069 INFO:tasks.rados.rados.0.smithi110.stdout:350: finishing write tid 5 to smithi11075562-13 2024-09-14T03:38:36.069 INFO:tasks.rados.rados.0.smithi110.stdout:346: done (4 left) 2024-09-14T03:38:36.069 INFO:tasks.rados.rados.0.smithi110.stdout:348: done (3 left) 2024-09-14T03:38:36.070 INFO:tasks.rados.rados.0.smithi110.stdout:349: done (2 left) 2024-09-14T03:38:36.070 INFO:tasks.rados.rados.0.smithi110.stdout:352: snap_create 2024-09-14T03:38:36.070 INFO:tasks.rados.rados.0.smithi110.stdout:350: finishing write tid 6 to smithi11075562-13 2024-09-14T03:38:36.070 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 107 (ObjNum 127 snap 33 seq_num 127) dirty exists 2024-09-14T03:38:36.070 INFO:tasks.rados.rados.0.smithi110.stdout:350: left oid 13 (ObjNum 127 snap 33 seq_num 127) 2024-09-14T03:38:36.071 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 116 (ObjNum 305 snap 0 seq_num 1) dirty exists 2024-09-14T03:38:37.057 INFO:tasks.rados.rados.0.smithi110.stdout:350: done (2 left) 2024-09-14T03:38:37.057 INFO:tasks.rados.rados.0.smithi110.stdout:351: done (1 left) 2024-09-14T03:38:37.057 INFO:tasks.rados.rados.0.smithi110.stdout:352: done (0 left) 2024-09-14T03:38:37.057 INFO:tasks.rados.rados.0.smithi110.stdout:353: copy_from oid 34 from oid 46 current snap is 34 2024-09-14T03:38:37.057 INFO:tasks.rados.rados.0.smithi110.stdout:354: snap_remove snap 31 2024-09-14T03:38:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: pgmap v183: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:38:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: Upgrade: Updating mgr.x 2024-09-14T03:38:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:38:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:38:37.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:38:37.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:37.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:37 smithi195 ceph-mon[30695]: Deploying daemon mgr.x on smithi195 2024-09-14T03:38:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: pgmap v183: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:38:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: Upgrade: Updating mgr.x 2024-09-14T03:38:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[35374]: Deploying daemon mgr.x on smithi195 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: pgmap v183: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: Upgrade: Updating mgr.x 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:38:37.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:38:37.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:38:37.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:37.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:37 smithi110 ceph-mon[27365]: Deploying daemon mgr.x on smithi195 2024-09-14T03:38:37.958 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:37 smithi195 systemd[1]: Stopping Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:354: done (1 left) 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:355: rollback oid 45 current snap is 34 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 28 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:356: read oid 20 snap -1 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:356: expect deleted 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:357: read oid 37 snap -1 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:357: expect deleted 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:358: delete oid 28 current snap is 34 2024-09-14T03:38:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:353: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:38:38.064 INFO:tasks.rados.rados.0.smithi110.stdout:353: finishing copy_from to smithi11075562-34 2024-09-14T03:38:38.064 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 42 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:38:38.066 INFO:tasks.rados.rados.0.smithi110.stdout:353: done (4 left) 2024-09-14T03:38:38.066 INFO:tasks.rados.rados.0.smithi110.stdout:356: done (3 left) 2024-09-14T03:38:38.066 INFO:tasks.rados.rados.0.smithi110.stdout:358: done (2 left) 2024-09-14T03:38:38.066 INFO:tasks.rados.rados.0.smithi110.stdout:359: rollback oid 13 current snap is 34 2024-09-14T03:38:38.066 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 30 2024-09-14T03:38:38.066 INFO:tasks.rados.rados.0.smithi110.stdout:360: snap_remove snap 16 2024-09-14T03:38:38.261 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:38 smithi195 ceph-mon[30695]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:38:38.261 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:37 smithi195 podman[63836]: 2024-09-14 03:38:37.963092248 +0000 UTC m=+0.138753605 container died 68be7f1bb79757065be1be289ba9f739d989c1f6ad4181f5db818351f8f353ab (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=quincy-eb16b97) 2024-09-14T03:38:38.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:38 smithi110 ceph-mon[35374]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:38:38.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:38 smithi110 ceph-mon[27365]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:38:38.603 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:38 smithi195 podman[63836]: 2024-09-14 03:38:38.350850092 +0000 UTC m=+0.526511452 container cleanup 68be7f1bb79757065be1be289ba9f739d989c1f6ad4181f5db818351f8f353ab (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-14T03:38:38.603 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:38 smithi195 bash[63836]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:38:38.924 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:38 smithi195 podman[63847]: 2024-09-14 03:38:38.602167497 +0000 UTC m=+0.631673670 container remove 68be7f1bb79757065be1be289ba9f739d989c1f6ad4181f5db818351f8f353ab (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T03:38:38.925 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:38 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:38:38.925 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:38 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-14T03:38:38.925 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:38 smithi195 systemd[1]: Stopped Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:38:38.925 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:38 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x.service: Consumed 10.113s CPU time. 2024-09-14T03:38:38.925 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:38 smithi195 systemd[1]: Starting Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:38:39.064 INFO:tasks.rados.rados.0.smithi110.stdout:360: done (3 left) 2024-09-14T03:38:39.065 INFO:tasks.rados.rados.0.smithi110.stdout:361: copy_from oid 18 from oid 30 current snap is 34 2024-09-14T03:38:39.065 INFO:tasks.rados.rados.0.smithi110.stdout:362: rmattr oid 5 current snap is 34 2024-09-14T03:38:39.065 INFO:tasks.rados.rados.0.smithi110.stdout:362: done (4 left) 2024-09-14T03:38:39.065 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 34 2024-09-14T03:38:39.065 INFO:tasks.rados.rados.0.smithi110.stdout:363: seq_num 129 ranges {2277376=565248,2842624=679936,3522560=786432,4308992=139264} 2024-09-14T03:38:39.081 INFO:tasks.rados.rados.0.smithi110.stdout:363: writing smithi11075562-25 from 2277376 to 2842624 tid 1 2024-09-14T03:38:39.086 INFO:tasks.rados.rados.0.smithi110.stdout:363: writing smithi11075562-25 from 2842624 to 3522560 tid 2 2024-09-14T03:38:39.090 INFO:tasks.rados.rados.0.smithi110.stdout:363: writing smithi11075562-25 from 3522560 to 4308992 tid 3 2024-09-14T03:38:39.091 INFO:tasks.rados.rados.0.smithi110.stdout:363: writing smithi11075562-25 from 4308992 to 4448256 tid 4 2024-09-14T03:38:39.091 INFO:tasks.rados.rados.0.smithi110.stdout:364: snap_create 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:355: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:355: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:359: finishing rollback tid 0 to smithi11075562-13 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:359: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:355: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 159 (ObjNum 110 snap 26 seq_num 110) dirty exists 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:359: finishing rollback tid 2 to smithi11075562-13 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 115 (ObjNum 80 snap 13 seq_num 80) dirty exists 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:363: finishing write tid 1 to smithi11075562-25 2024-09-14T03:38:39.092 INFO:tasks.rados.rados.0.smithi110.stdout:361: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:38:39.093 INFO:tasks.rados.rados.0.smithi110.stdout:361: finishing copy_from to smithi11075562-18 2024-09-14T03:38:39.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 42 (ObjNum 93 snap 19 seq_num 93) dirty exists 2024-09-14T03:38:39.093 INFO:tasks.rados.rados.0.smithi110.stdout:363: finishing write tid 2 to smithi11075562-25 2024-09-14T03:38:39.094 INFO:tasks.rados.rados.0.smithi110.stdout:363: finishing write tid 3 to smithi11075562-25 2024-09-14T03:38:39.094 INFO:tasks.rados.rados.0.smithi110.stdout:363: finishing write tid 4 to smithi11075562-25 2024-09-14T03:38:39.094 INFO:tasks.rados.rados.0.smithi110.stdout:363: finishing write tid 5 to smithi11075562-25 2024-09-14T03:38:39.095 INFO:tasks.rados.rados.0.smithi110.stdout:363: finishing write tid 6 to smithi11075562-25 2024-09-14T03:38:39.095 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 175 (ObjNum 129 snap 34 seq_num 129) dirty exists 2024-09-14T03:38:39.095 INFO:tasks.rados.rados.0.smithi110.stdout:363: left oid 25 (ObjNum 129 snap 34 seq_num 129) 2024-09-14T03:38:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:39 smithi195 ceph-mon[30695]: pgmap v186: 121 pgs: 121 active+clean; 96 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 MiB/s wr, 18 op/s 2024-09-14T03:38:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:39 smithi195 ceph-mon[30695]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:38:39.209 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:39 smithi195 podman[63980]: 2024-09-14 03:38:39.034767492 +0000 UTC m=+0.015973988 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:38:39.209 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:39 smithi195 podman[63980]: 2024-09-14 03:38:39.170142051 +0000 UTC m=+0.151348585 container create 2372ccbce64cc806b61b00f1a2cc939df316b2d640c7c825e84d717b88440b8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T03:38:39.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:39 smithi110 ceph-mon[35374]: pgmap v186: 121 pgs: 121 active+clean; 96 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 MiB/s wr, 18 op/s 2024-09-14T03:38:39.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:39 smithi110 ceph-mon[35374]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:38:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:39 smithi110 ceph-mon[27365]: pgmap v186: 121 pgs: 121 active+clean; 96 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 MiB/s wr, 18 op/s 2024-09-14T03:38:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:39 smithi110 ceph-mon[27365]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:38:39.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:39 smithi195 podman[63980]: 2024-09-14 03:38:39.397525356 +0000 UTC m=+0.378731851 container init 2372ccbce64cc806b61b00f1a2cc939df316b2d640c7c825e84d717b88440b8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:38:39.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:39 smithi195 podman[63980]: 2024-09-14 03:38:39.400302438 +0000 UTC m=+0.381508933 container start 2372ccbce64cc806b61b00f1a2cc939df316b2d640c7c825e84d717b88440b8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T03:38:39.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:39 smithi195 ceph-mgr[63999]: -- 172.21.15.195:0/1571971149 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fb7b20d0e0 con 0x55fb7b1eb400 2024-09-14T03:38:39.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:39 smithi195 bash[63980]: 2372ccbce64cc806b61b00f1a2cc939df316b2d640c7c825e84d717b88440b8a 2024-09-14T03:38:39.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:39 smithi195 systemd[1]: Started Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:38:40.020 INFO:tasks.rados.rados.0.smithi110.stdout:355: done (5 left) 2024-09-14T03:38:40.020 INFO:tasks.rados.rados.0.smithi110.stdout:357: done (4 left) 2024-09-14T03:38:40.020 INFO:tasks.rados.rados.0.smithi110.stdout:359: done (3 left) 2024-09-14T03:38:40.020 INFO:tasks.rados.rados.0.smithi110.stdout:361: done (2 left) 2024-09-14T03:38:40.021 INFO:tasks.rados.rados.0.smithi110.stdout:363: done (1 left) 2024-09-14T03:38:40.021 INFO:tasks.rados.rados.0.smithi110.stdout:364: done (0 left) 2024-09-14T03:38:40.021 INFO:tasks.rados.rados.0.smithi110.stdout:365: read oid 25 snap 30 2024-09-14T03:38:40.021 INFO:tasks.rados.rados.0.smithi110.stdout:365: expect (ObjNum 99 snap 20 seq_num 99) 2024-09-14T03:38:40.021 INFO:tasks.rados.rados.0.smithi110.stdout:366: snap_remove snap 28 2024-09-14T03:38:40.281 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:38:40.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:40 smithi195 ceph-mon[30695]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:38:40.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:40 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:40.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:40 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:40.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:40 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:40.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:40 smithi195 ceph-mon[30695]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:38:40.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[35374]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:38:40.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:40.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:40.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:40.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[35374]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:38:40.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[27365]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:38:40.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:40.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:40.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:40.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-mon[27365]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:38:40.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:40 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:40] "GET /metrics HTTP/1.1" 200 239995 "" "Prometheus/2.43.0" 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (2m) 2m ago 3m 18.3M - 0.25.0 c8568f914cd2 764997e49f35 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (3m) 2m ago 3m 80.5M - 9.4.7 954c08fa6188 67ea5066f839 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (4m) 2m ago 4m 16.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (4m) 2m ago 4m 15.0M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283 starting - - - - 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:9283,8765 running (9m) 2m ago 9m 446M - 17.2.7-1629-geb16b97b ab51d1bebd12 a83ea41d9e62 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (9m) 2m ago 10m 44.8M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 c1073676c66c 2024-09-14T03:38:40.786 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (8m) 2m ago 8m 40.7M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 b6415a0de807 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (8m) 2m ago 8m 39.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 8e2894672cd8 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (3m) 2m ago 3m 13.2M - 1.5.0 0da6a335fe13 986afca49027 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (3m) 2m ago 3m 3359k - 1.5.0 0da6a335fe13 67a66167cf52 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (7m) 2m ago 7m 43.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (7m) 2m ago 7m 43.8M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (7m) 2m ago 7m 43.1M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (6m) 2m ago 6m 41.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (6m) 2m ago 6m 41.7M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (6m) 2m ago 5m 42.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (5m) 2m ago 5m 41.5M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (5m) 2m ago 5m 42.8M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:38:40.787 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (2m) 2m ago 3m 24.8M - 2.43.0 a07b618ecd1d 4460d604b8f6 2024-09-14T03:38:41.024 INFO:tasks.rados.rados.0.smithi110.stdout:366: done (1 left) 2024-09-14T03:38:41.024 INFO:tasks.rados.rados.0.smithi110.stdout:367: copy_from oid 9 from oid 36 current snap is 35 2024-09-14T03:38:41.024 INFO:tasks.rados.rados.0.smithi110.stdout:368: snap_create 2024-09-14T03:38:41.024 INFO:tasks.rados.rados.0.smithi110.stdout:365: expect (ObjNum 99 snap 20 seq_num 99) 2024-09-14T03:38:41.095 INFO:tasks.rados.rados.0.smithi110.stdout:367: finishing copy_from racing read to smithi11075562-9 2024-09-14T03:38:41.096 INFO:tasks.rados.rados.0.smithi110.stdout:367: finishing copy_from to smithi11075562-9 2024-09-14T03:38:41.096 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 101 (ObjNum 104 snap 20 seq_num 104) dirty exists 2024-09-14T03:38:41.100 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:38:41.100 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:38:41.100 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:38:41.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 15 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:38:41.101 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:38:41.356 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:38:41.357 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:38:41.357 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:38:41.357 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:38:41.357 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [], 2024-09-14T03:38:41.357 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "", 2024-09-14T03:38:41.357 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading mgr daemons", 2024-09-14T03:38:41.357 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:38:41.357 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:38:41.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:41 smithi110 ceph-mon[27365]: pgmap v189: 121 pgs: 121 active+clean; 96 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 MiB/s wr, 18 op/s 2024-09-14T03:38:41.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:41 smithi110 ceph-mon[27365]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:41.358 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:41 smithi110 ceph-mon[27365]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:38:41.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:41 smithi110 ceph-mon[35374]: pgmap v189: 121 pgs: 121 active+clean; 96 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 MiB/s wr, 18 op/s 2024-09-14T03:38:41.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:41 smithi110 ceph-mon[35374]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:41.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:41 smithi110 ceph-mon[35374]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:38:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:41 smithi195 ceph-mon[30695]: pgmap v189: 121 pgs: 121 active+clean; 96 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 MiB/s wr, 18 op/s 2024-09-14T03:38:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:41 smithi195 ceph-mon[30695]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:41 smithi195 ceph-mon[30695]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:38:42.022 INFO:tasks.rados.rados.0.smithi110.stdout:365: done (2 left) 2024-09-14T03:38:42.022 INFO:tasks.rados.rados.0.smithi110.stdout:367: done (1 left) 2024-09-14T03:38:42.022 INFO:tasks.rados.rados.0.smithi110.stdout:368: done (0 left) 2024-09-14T03:38:42.022 INFO:tasks.rados.rados.0.smithi110.stdout:369: snap_remove snap 34 2024-09-14T03:38:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[35374]: from='client.24884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[35374]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2273097718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:38:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[35374]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[35374]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:38:42.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[27365]: from='client.24884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[27365]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2273097718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:38:42.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[27365]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:42 smithi110 ceph-mon[27365]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:38:42.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:42 smithi195 ceph-mon[30695]: from='client.24884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:42 smithi195 ceph-mon[30695]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:42 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2273097718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:38:42.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:42 smithi195 ceph-mon[30695]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:42.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:42 smithi195 ceph-mon[30695]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:38:43.024 INFO:tasks.rados.rados.0.smithi110.stdout:369: done (0 left) 2024-09-14T03:38:43.024 INFO:tasks.rados.rados.0.smithi110.stdout:370: snap_remove snap 22 2024-09-14T03:38:43.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:43 smithi195 ceph-mon[30695]: pgmap v192: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:38:43.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:43 smithi195 ceph-mon[30695]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:38:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:43 smithi110 ceph-mon[35374]: pgmap v192: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:38:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:43 smithi110 ceph-mon[35374]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:38:43.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:43 smithi110 ceph-mon[27365]: pgmap v192: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:38:43.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:43 smithi110 ceph-mon[27365]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:38:44.026 INFO:tasks.rados.rados.0.smithi110.stdout:370: done (0 left) 2024-09-14T03:38:44.026 INFO:tasks.rados.rados.0.smithi110.stdout:371: copy_from oid 43 from oid 3 current snap is 36 2024-09-14T03:38:44.026 INFO:tasks.rados.rados.0.smithi110.stdout:372: read oid 37 snap -1 2024-09-14T03:38:44.026 INFO:tasks.rados.rados.0.smithi110.stdout:372: expect deleted 2024-09-14T03:38:44.026 INFO:tasks.rados.rados.0.smithi110.stdout:373: delete oid 12 current snap is 36 2024-09-14T03:38:44.033 INFO:tasks.rados.rados.0.smithi110.stdout:372: done (2 left) 2024-09-14T03:38:44.033 INFO:tasks.rados.rados.0.smithi110.stdout:373: done (1 left) 2024-09-14T03:38:44.033 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 36 2024-09-14T03:38:44.033 INFO:tasks.rados.rados.0.smithi110.stdout:374: seq_num 131 ranges {2072576=598016,2670592=499712,3170304=753664,3923968=352256} 2024-09-14T03:38:44.047 INFO:tasks.rados.rados.0.smithi110.stdout:374: writing smithi11075562-5 from 2072576 to 2670592 tid 1 2024-09-14T03:38:44.050 INFO:tasks.rados.rados.0.smithi110.stdout:374: writing smithi11075562-5 from 2670592 to 3170304 tid 2 2024-09-14T03:38:44.054 INFO:tasks.rados.rados.0.smithi110.stdout:374: writing smithi11075562-5 from 3170304 to 3923968 tid 3 2024-09-14T03:38:44.056 INFO:tasks.rados.rados.0.smithi110.stdout:374: writing smithi11075562-5 from 3923968 to 4276224 tid 4 2024-09-14T03:38:44.056 INFO:tasks.rados.rados.0.smithi110.stdout:375: delete oid 16 current snap is 36 2024-09-14T03:38:44.056 INFO:tasks.rados.rados.0.smithi110.stdout:371: finishing copy_from racing read to smithi11075562-43 2024-09-14T03:38:44.057 INFO:tasks.rados.rados.0.smithi110.stdout:371: finishing copy_from to smithi11075562-43 2024-09-14T03:38:44.057 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 8 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:38:44.057 INFO:tasks.rados.rados.0.smithi110.stdout:374: finishing write tid 1 to smithi11075562-5 2024-09-14T03:38:44.057 INFO:tasks.rados.rados.0.smithi110.stdout:374: finishing write tid 2 to smithi11075562-5 2024-09-14T03:38:44.058 INFO:tasks.rados.rados.0.smithi110.stdout:371: done (2 left) 2024-09-14T03:38:44.058 INFO:tasks.rados.rados.0.smithi110.stdout:375: done (1 left) 2024-09-14T03:38:44.058 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 36 2024-09-14T03:38:44.058 INFO:tasks.rados.rados.0.smithi110.stdout:376: seq_num 133 ranges {3293184=630784,3923968=663552,4587520=770048,5357568=172032} 2024-09-14T03:38:44.080 INFO:tasks.rados.rados.0.smithi110.stdout:376: writing smithi11075562-36 from 3293184 to 3923968 tid 1 2024-09-14T03:38:44.083 INFO:tasks.rados.rados.0.smithi110.stdout:376: writing smithi11075562-36 from 3923968 to 4587520 tid 2 2024-09-14T03:38:44.088 INFO:tasks.rados.rados.0.smithi110.stdout:376: writing smithi11075562-36 from 4587520 to 5357568 tid 3 2024-09-14T03:38:44.089 INFO:tasks.rados.rados.0.smithi110.stdout:376: writing smithi11075562-36 from 5357568 to 5529600 tid 4 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:377: copy_from oid 21 from oid 20 current snap is 36 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:374: finishing write tid 3 to smithi11075562-5 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:374: finishing write tid 4 to smithi11075562-5 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:374: finishing write tid 5 to smithi11075562-5 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:374: finishing write tid 6 to smithi11075562-5 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 192 (ObjNum 131 snap 36 seq_num 131) dirty exists 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:374: left oid 5 (ObjNum 131 snap 36 seq_num 131) 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:376: finishing write tid 1 to smithi11075562-36 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:376: finishing write tid 2 to smithi11075562-36 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:374: done (2 left) 2024-09-14T03:38:44.090 INFO:tasks.rados.rados.0.smithi110.stdout:378: setattr oid 43 current snap is 36 2024-09-14T03:38:44.091 INFO:tasks.rados.rados.0.smithi110.stdout:379: read oid 34 snap -1 2024-09-14T03:38:44.091 INFO:tasks.rados.rados.0.smithi110.stdout:379: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:38:44.091 INFO:tasks.rados.rados.0.smithi110.stdout:380: delete oid 10 current snap is 36 2024-09-14T03:38:44.091 INFO:tasks.rados.rados.0.smithi110.stdout:377: finishing copy_from to smithi11075562-21 2024-09-14T03:38:44.091 INFO:tasks.rados.rados.0.smithi110.stdout:377: got expected ENOENT (src dne) 2024-09-14T03:38:44.092 INFO:tasks.rados.rados.0.smithi110.stdout:377: finishing copy_from racing read to smithi11075562-21 2024-09-14T03:38:44.092 INFO:tasks.rados.rados.0.smithi110.stdout:376: finishing write tid 3 to smithi11075562-36 2024-09-14T03:38:44.092 INFO:tasks.rados.rados.0.smithi110.stdout:376: finishing write tid 4 to smithi11075562-36 2024-09-14T03:38:44.092 INFO:tasks.rados.rados.0.smithi110.stdout:376: finishing write tid 5 to smithi11075562-36 2024-09-14T03:38:44.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 145 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:38:44.093 INFO:tasks.rados.rados.0.smithi110.stdout:376: finishing write tid 6 to smithi11075562-36 2024-09-14T03:38:44.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 198 (ObjNum 133 snap 36 seq_num 133) dirty exists 2024-09-14T03:38:44.093 INFO:tasks.rados.rados.0.smithi110.stdout:376: left oid 36 (ObjNum 133 snap 36 seq_num 133) 2024-09-14T03:38:44.096 INFO:tasks.rados.rados.0.smithi110.stdout:376: done (4 left) 2024-09-14T03:38:44.096 INFO:tasks.rados.rados.0.smithi110.stdout:377: done (3 left) 2024-09-14T03:38:44.096 INFO:tasks.rados.rados.0.smithi110.stdout:378: done (2 left) 2024-09-14T03:38:44.096 INFO:tasks.rados.rados.0.smithi110.stdout:380: done (1 left) 2024-09-14T03:38:44.096 INFO:tasks.rados.rados.0.smithi110.stdout:381: rollback oid 46 current snap is 36 2024-09-14T03:38:44.097 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 46 to 32 2024-09-14T03:38:44.097 INFO:tasks.rados.rados.0.smithi110.stdout:382: rmattr oid 9 current snap is 36 2024-09-14T03:38:44.097 INFO:tasks.rados.rados.0.smithi110.stdout:383: snap_remove snap 29 2024-09-14T03:38:44.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:38:44] ENGINE Bus STARTING 2024-09-14T03:38:44.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: CherryPy Checker: 2024-09-14T03:38:44.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: The Application mounted at '' has an empty config. 2024-09-14T03:38:44.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:38:44.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:38:44] ENGINE Serving on http://:::9283 2024-09-14T03:38:44.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:38:44] ENGINE Bus STARTED 2024-09-14T03:38:45.022 INFO:tasks.rados.rados.0.smithi110.stdout:379: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:38:45.045 INFO:tasks.rados.rados.0.smithi110.stdout:381: finishing rollback tid 0 to smithi11075562-46 2024-09-14T03:38:45.045 INFO:tasks.rados.rados.0.smithi110.stdout:381: finishing rollback tid 1 to smithi11075562-46 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:381: finishing rollback tid 2 to smithi11075562-46 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 129 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 0 (ObjNum 104 snap 20 seq_num 104) dirty exists 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:379: done (3 left) 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:381: done (2 left) 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:382: done (1 left) 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:383: done (0 left) 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:384: copy_from oid 31 from oid 44 current snap is 36 2024-09-14T03:38:45.046 INFO:tasks.rados.rados.0.smithi110.stdout:385: snap_create 2024-09-14T03:38:45.049 INFO:tasks.rados.rados.0.smithi110.stdout:384: finishing copy_from to smithi11075562-31 2024-09-14T03:38:45.049 INFO:tasks.rados.rados.0.smithi110.stdout:384: got expected ENOENT (src dne) 2024-09-14T03:38:45.050 INFO:tasks.rados.rados.0.smithi110.stdout:384: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:38:45.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: pgmap v195: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:38:45.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:38:45.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: Standby manager daemon x restarted 2024-09-14T03:38:45.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: Standby manager daemon x started 2024-09-14T03:38:45.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:38:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:38:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:38:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:38:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[35374]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:38:45.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: pgmap v195: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:38:45.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:38:45.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: Standby manager daemon x restarted 2024-09-14T03:38:45.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: Standby manager daemon x started 2024-09-14T03:38:45.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:38:45.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:38:45.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:38:45.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:38:45.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:45 smithi110 ceph-mon[27365]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: pgmap v195: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: Standby manager daemon x restarted 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: Standby manager daemon x started 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:38:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:45 smithi195 ceph-mon[30695]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:38:46.030 INFO:tasks.rados.rados.0.smithi110.stdout:384: done (1 left) 2024-09-14T03:38:46.030 INFO:tasks.rados.rados.0.smithi110.stdout:385: done (0 left) 2024-09-14T03:38:46.030 INFO:tasks.rados.rados.0.smithi110.stdout:386: copy_from oid 27 from oid 39 current snap is 37 2024-09-14T03:38:46.030 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 37 2024-09-14T03:38:46.030 INFO:tasks.rados.rados.0.smithi110.stdout:387: seq_num 135 ranges {0=663552,663552=434176,1097728=786432,1884160=385024} 2024-09-14T03:38:46.033 INFO:tasks.rados.rados.0.smithi110.stdout:387: writing smithi11075562-8 from 0 to 663552 tid 1 2024-09-14T03:38:46.036 INFO:tasks.rados.rados.0.smithi110.stdout:387: writing smithi11075562-8 from 663552 to 1097728 tid 2 2024-09-14T03:38:46.041 INFO:tasks.rados.rados.0.smithi110.stdout:387: writing smithi11075562-8 from 1097728 to 1884160 tid 3 2024-09-14T03:38:46.044 INFO:tasks.rados.rados.0.smithi110.stdout:387: writing smithi11075562-8 from 1884160 to 2269184 tid 4 2024-09-14T03:38:46.044 INFO:tasks.rados.rados.0.smithi110.stdout:388: copy_from oid 45 from oid 50 current snap is 37 2024-09-14T03:38:46.044 INFO:tasks.rados.rados.0.smithi110.stdout:386: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:38:46.044 INFO:tasks.rados.rados.0.smithi110.stdout:386: finishing copy_from to smithi11075562-27 2024-09-14T03:38:46.044 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 44 (ObjNum 66 snap 6 seq_num 66) dirty exists 2024-09-14T03:38:46.044 INFO:tasks.rados.rados.0.smithi110.stdout:387: finishing write tid 1 to smithi11075562-8 2024-09-14T03:38:46.044 INFO:tasks.rados.rados.0.smithi110.stdout:387: finishing write tid 2 to smithi11075562-8 2024-09-14T03:38:46.045 INFO:tasks.rados.rados.0.smithi110.stdout:386: done (2 left) 2024-09-14T03:38:46.045 INFO:tasks.rados.rados.0.smithi110.stdout:389: snap_create 2024-09-14T03:38:46.045 INFO:tasks.rados.rados.0.smithi110.stdout:387: finishing write tid 3 to smithi11075562-8 2024-09-14T03:38:46.046 INFO:tasks.rados.rados.0.smithi110.stdout:387: finishing write tid 4 to smithi11075562-8 2024-09-14T03:38:46.046 INFO:tasks.rados.rados.0.smithi110.stdout:387: finishing write tid 5 to smithi11075562-8 2024-09-14T03:38:46.047 INFO:tasks.rados.rados.0.smithi110.stdout:387: finishing write tid 6 to smithi11075562-8 2024-09-14T03:38:46.047 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 113 (ObjNum 135 snap 37 seq_num 135) dirty exists 2024-09-14T03:38:46.047 INFO:tasks.rados.rados.0.smithi110.stdout:387: left oid 8 (ObjNum 135 snap 37 seq_num 135) 2024-09-14T03:38:46.052 INFO:tasks.rados.rados.0.smithi110.stdout:388: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:38:46.053 INFO:tasks.rados.rados.0.smithi110.stdout:388: finishing copy_from to smithi11075562-45 2024-09-14T03:38:46.053 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 7 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:38:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:46 smithi110 ceph-mon[35374]: mgrmap e23: y(active, since 3m), standbys: x 2024-09-14T03:38:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:46 smithi110 ceph-mon[35374]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:38:46.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:46 smithi110 ceph-mon[27365]: mgrmap e23: y(active, since 3m), standbys: x 2024-09-14T03:38:46.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:46 smithi110 ceph-mon[27365]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:38:46.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:46 smithi195 ceph-mon[30695]: mgrmap e23: y(active, since 3m), standbys: x 2024-09-14T03:38:46.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:46 smithi195 ceph-mon[30695]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:38:47.031 INFO:tasks.rados.rados.0.smithi110.stdout:387: done (2 left) 2024-09-14T03:38:47.031 INFO:tasks.rados.rados.0.smithi110.stdout:388: done (1 left) 2024-09-14T03:38:47.032 INFO:tasks.rados.rados.0.smithi110.stdout:389: done (0 left) 2024-09-14T03:38:47.032 INFO:tasks.rados.rados.0.smithi110.stdout:390: read oid 36 snap -1 2024-09-14T03:38:47.032 INFO:tasks.rados.rados.0.smithi110.stdout:390: expect (ObjNum 133 snap 36 seq_num 133) 2024-09-14T03:38:47.032 INFO:tasks.rados.rados.0.smithi110.stdout:391: delete oid 21 current snap is 38 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:391: done (1 left) 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:392: rmattr oid 9 current snap is 38 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:392: done (1 left) 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:393: rmattr oid 32 current snap is 38 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:393: done (1 left) 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:394: rmattr oid 6 current snap is 38 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:394: done (1 left) 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:395: copy_from oid 15 from oid 19 current snap is 38 2024-09-14T03:38:47.036 INFO:tasks.rados.rados.0.smithi110.stdout:396: delete oid 3 current snap is 38 2024-09-14T03:38:47.038 INFO:tasks.rados.rados.0.smithi110.stdout:396: done (2 left) 2024-09-14T03:38:47.038 INFO:tasks.rados.rados.0.smithi110.stdout:397: rollback oid 49 current snap is 38 2024-09-14T03:38:47.038 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 20 2024-09-14T03:38:47.039 INFO:tasks.rados.rados.0.smithi110.stdout:398: read oid 45 snap -1 2024-09-14T03:38:47.039 INFO:tasks.rados.rados.0.smithi110.stdout:398: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:38:47.039 INFO:tasks.rados.rados.0.smithi110.stdout:399: snap_remove snap 33 2024-09-14T03:38:47.439 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:47 smithi195 ceph-mon[30695]: pgmap v198: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:47.439 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:47 smithi195 ceph-mon[30695]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:38:47.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:47 smithi110 ceph-mon[35374]: pgmap v198: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:47.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:47 smithi110 ceph-mon[35374]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:38:47.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:47 smithi110 ceph-mon[27365]: pgmap v198: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:47.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:47 smithi110 ceph-mon[27365]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:38:48.037 INFO:tasks.rados.rados.0.smithi110.stdout:399: done (4 left) 2024-09-14T03:38:48.037 INFO:tasks.rados.rados.0.smithi110.stdout:400: delete oid 5 current snap is 38 2024-09-14T03:38:48.037 INFO:tasks.rados.rados.0.smithi110.stdout:397: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:38:48.037 INFO:tasks.rados.rados.0.smithi110.stdout:397: finishing rollback tid 2 to smithi11075562-49 2024-09-14T03:38:48.038 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 139 (ObjNum 98 snap 20 seq_num 98) dirty exists 2024-09-14T03:38:48.038 INFO:tasks.rados.rados.0.smithi110.stdout:395: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:38:48.038 INFO:tasks.rados.rados.0.smithi110.stdout:390: expect (ObjNum 133 snap 36 seq_num 133) 2024-09-14T03:38:48.255 INFO:tasks.rados.rados.0.smithi110.stdout:395: finishing copy_from to smithi11075562-15 2024-09-14T03:38:48.255 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 119 (ObjNum 126 snap 33 seq_num 126) dirty exists 2024-09-14T03:38:48.255 INFO:tasks.rados.rados.0.smithi110.stdout:398: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:38:48.275 INFO:tasks.rados.rados.0.smithi110.stdout:390: done (4 left) 2024-09-14T03:38:48.275 INFO:tasks.rados.rados.0.smithi110.stdout:395: done (3 left) 2024-09-14T03:38:48.276 INFO:tasks.rados.rados.0.smithi110.stdout:397: done (2 left) 2024-09-14T03:38:48.276 INFO:tasks.rados.rados.0.smithi110.stdout:398: done (1 left) 2024-09-14T03:38:48.276 INFO:tasks.rados.rados.0.smithi110.stdout:400: done (0 left) 2024-09-14T03:38:48.276 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 38 2024-09-14T03:38:48.276 INFO:tasks.rados.rados.0.smithi110.stdout:401: seq_num 139 ranges {0=737280,737280=425984,1163264=761856,1925120=417792} 2024-09-14T03:38:48.279 INFO:tasks.rados.rados.0.smithi110.stdout:401: writing smithi11075562-12 from 0 to 737280 tid 1 2024-09-14T03:38:48.282 INFO:tasks.rados.rados.0.smithi110.stdout:401: writing smithi11075562-12 from 737280 to 1163264 tid 2 2024-09-14T03:38:48.286 INFO:tasks.rados.rados.0.smithi110.stdout:401: writing smithi11075562-12 from 1163264 to 1925120 tid 3 2024-09-14T03:38:48.289 INFO:tasks.rados.rados.0.smithi110.stdout:401: writing smithi11075562-12 from 1925120 to 2342912 tid 4 2024-09-14T03:38:48.289 INFO:tasks.rados.rados.0.smithi110.stdout:402: snap_remove snap 37 2024-09-14T03:38:49.090 INFO:tasks.rados.rados.0.smithi110.stdout:402: done (1 left) 2024-09-14T03:38:49.090 INFO:tasks.rados.rados.0.smithi110.stdout:403: snap_remove snap 35 2024-09-14T03:38:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:49 smithi110 ceph-mon[35374]: pgmap v201: 121 pgs: 121 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 563 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:38:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:49 smithi110 ceph-mon[35374]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:38:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:49 smithi110 ceph-mon[27365]: pgmap v201: 121 pgs: 121 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 563 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:38:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:49 smithi110 ceph-mon[27365]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:38:49.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:49 smithi195 ceph-mon[30695]: pgmap v201: 121 pgs: 121 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 563 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:38:49.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:49 smithi195 ceph-mon[30695]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:38:50.025 INFO:tasks.rados.rados.0.smithi110.stdout:403: done (1 left) 2024-09-14T03:38:50.025 INFO:tasks.rados.rados.0.smithi110.stdout:404: snap_remove snap 36 2024-09-14T03:38:50.026 INFO:tasks.rados.rados.0.smithi110.stdout:401: finishing write tid 1 to smithi11075562-12 2024-09-14T03:38:50.026 INFO:tasks.rados.rados.0.smithi110.stdout:401: finishing write tid 2 to smithi11075562-12 2024-09-14T03:38:50.026 INFO:tasks.rados.rados.0.smithi110.stdout:401: finishing write tid 3 to smithi11075562-12 2024-09-14T03:38:50.026 INFO:tasks.rados.rados.0.smithi110.stdout:401: finishing write tid 4 to smithi11075562-12 2024-09-14T03:38:50.026 INFO:tasks.rados.rados.0.smithi110.stdout:401: finishing write tid 5 to smithi11075562-12 2024-09-14T03:38:50.026 INFO:tasks.rados.rados.0.smithi110.stdout:401: finishing write tid 6 to smithi11075562-12 2024-09-14T03:38:50.026 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 221 (ObjNum 139 snap 38 seq_num 139) dirty exists 2024-09-14T03:38:50.026 INFO:tasks.rados.rados.0.smithi110.stdout:401: left oid 12 (ObjNum 139 snap 38 seq_num 139) 2024-09-14T03:38:50.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:50 smithi110 ceph-mon[35374]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:38:50.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:50 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:50.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:50 smithi110 ceph-mon[35374]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:38:50.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:50 smithi110 ceph-mon[27365]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:38:50.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:50 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:50.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:50 smithi110 ceph-mon[27365]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:38:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:50 smithi195 ceph-mon[30695]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:38:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:50 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:38:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:50 smithi195 ceph-mon[30695]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:38:50.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:38:50.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:50 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:38:50] "GET /metrics HTTP/1.1" 200 240068 "" "Prometheus/2.43.0" 2024-09-14T03:38:51.026 INFO:tasks.rados.rados.0.smithi110.stdout:401: done (1 left) 2024-09-14T03:38:51.026 INFO:tasks.rados.rados.0.smithi110.stdout:404: done (0 left) 2024-09-14T03:38:51.026 INFO:tasks.rados.rados.0.smithi110.stdout:405: snap_remove snap 27 2024-09-14T03:38:51.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:51 smithi110 ceph-mon[35374]: pgmap v204: 121 pgs: 121 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 563 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:38:51.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:51 smithi110 ceph-mon[35374]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:38:51.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:51 smithi110 ceph-mon[27365]: pgmap v204: 121 pgs: 121 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 563 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:38:51.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:51 smithi110 ceph-mon[27365]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:38:51.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:51 smithi195 ceph-mon[30695]: pgmap v204: 121 pgs: 121 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 563 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:38:51.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:51 smithi195 ceph-mon[30695]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:38:52.028 INFO:tasks.rados.rados.0.smithi110.stdout:405: done (0 left) 2024-09-14T03:38:52.028 INFO:tasks.rados.rados.0.smithi110.stdout:406: setattr oid 41 current snap is 38 2024-09-14T03:38:52.033 INFO:tasks.rados.rados.0.smithi110.stdout:407: read oid 26 snap -1 2024-09-14T03:38:52.033 INFO:tasks.rados.rados.0.smithi110.stdout:407: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-14T03:38:52.033 INFO:tasks.rados.rados.0.smithi110.stdout:408: read oid 5 snap -1 2024-09-14T03:38:52.033 INFO:tasks.rados.rados.0.smithi110.stdout:408: expect deleted 2024-09-14T03:38:52.033 INFO:tasks.rados.rados.0.smithi110.stdout:409: read oid 27 snap 21 2024-09-14T03:38:52.033 INFO:tasks.rados.rados.0.smithi110.stdout:409: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-14T03:38:52.033 INFO:tasks.rados.rados.0.smithi110.stdout:410: setattr oid 6 current snap is 38 2024-09-14T03:38:52.037 INFO:tasks.rados.rados.0.smithi110.stdout:411: snap_create 2024-09-14T03:38:52.041 INFO:tasks.rados.rados.0.smithi110.stdout:407: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-14T03:38:52.061 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 168 (ObjNum 65 snap 0 seq_num 64595680) dirty exists 2024-09-14T03:38:52.061 INFO:tasks.rados.rados.0.smithi110.stdout:409: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-14T03:38:52.072 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 165 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-14T03:38:53.033 INFO:tasks.rados.rados.0.smithi110.stdout:406: done (5 left) 2024-09-14T03:38:53.033 INFO:tasks.rados.rados.0.smithi110.stdout:407: done (4 left) 2024-09-14T03:38:53.033 INFO:tasks.rados.rados.0.smithi110.stdout:408: done (3 left) 2024-09-14T03:38:53.033 INFO:tasks.rados.rados.0.smithi110.stdout:409: done (2 left) 2024-09-14T03:38:53.033 INFO:tasks.rados.rados.0.smithi110.stdout:410: done (1 left) 2024-09-14T03:38:53.034 INFO:tasks.rados.rados.0.smithi110.stdout:411: done (0 left) 2024-09-14T03:38:53.034 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 39 2024-09-14T03:38:53.034 INFO:tasks.rados.rados.0.smithi110.stdout:412: seq_num 140 ranges {1556480=753664,2310144=704512,3014656=524288,3538944=376832} 2024-09-14T03:38:53.044 INFO:tasks.rados.rados.0.smithi110.stdout:412: writing smithi11075562-13 from 1556480 to 2310144 tid 1 2024-09-14T03:38:53.048 INFO:tasks.rados.rados.0.smithi110.stdout:412: writing smithi11075562-13 from 2310144 to 3014656 tid 2 2024-09-14T03:38:53.051 INFO:tasks.rados.rados.0.smithi110.stdout:412: writing smithi11075562-13 from 3014656 to 3538944 tid 3 2024-09-14T03:38:53.054 INFO:tasks.rados.rados.0.smithi110.stdout:412: writing smithi11075562-13 from 3538944 to 3915776 tid 4 2024-09-14T03:38:53.054 INFO:tasks.rados.rados.0.smithi110.stdout:413: delete oid 37 current snap is 39 2024-09-14T03:38:53.054 INFO:tasks.rados.rados.0.smithi110.stdout:412: finishing write tid 1 to smithi11075562-13 2024-09-14T03:38:53.054 INFO:tasks.rados.rados.0.smithi110.stdout:412: finishing write tid 2 to smithi11075562-13 2024-09-14T03:38:53.054 INFO:tasks.rados.rados.0.smithi110.stdout:412: finishing write tid 3 to smithi11075562-13 2024-09-14T03:38:53.056 INFO:tasks.rados.rados.0.smithi110.stdout:413: done (1 left) 2024-09-14T03:38:53.056 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 39 2024-09-14T03:38:53.056 INFO:tasks.rados.rados.0.smithi110.stdout:414: seq_num 142 ranges {4472832=786432,5259264=475136,5734400=540672,6275072=589824} 2024-09-14T03:38:53.084 INFO:tasks.rados.rados.0.smithi110.stdout:414: writing smithi11075562-38 from 4472832 to 5259264 tid 1 2024-09-14T03:38:53.087 INFO:tasks.rados.rados.0.smithi110.stdout:414: writing smithi11075562-38 from 5259264 to 5734400 tid 2 2024-09-14T03:38:53.090 INFO:tasks.rados.rados.0.smithi110.stdout:414: writing smithi11075562-38 from 5734400 to 6275072 tid 3 2024-09-14T03:38:53.094 INFO:tasks.rados.rados.0.smithi110.stdout:414: writing smithi11075562-38 from 6275072 to 6864896 tid 4 2024-09-14T03:38:53.094 INFO:tasks.rados.rados.0.smithi110.stdout:415: setattr oid 20 current snap is 39 2024-09-14T03:38:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:412: finishing write tid 4 to smithi11075562-13 2024-09-14T03:38:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:412: finishing write tid 5 to smithi11075562-13 2024-09-14T03:38:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:412: finishing write tid 6 to smithi11075562-13 2024-09-14T03:38:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 153 (ObjNum 140 snap 39 seq_num 140) dirty exists 2024-09-14T03:38:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:412: left oid 13 (ObjNum 140 snap 39 seq_num 140) 2024-09-14T03:38:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:414: finishing write tid 1 to smithi11075562-38 2024-09-14T03:38:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:414: finishing write tid 2 to smithi11075562-38 2024-09-14T03:38:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:414: finishing write tid 3 to smithi11075562-38 2024-09-14T03:38:53.096 INFO:tasks.rados.rados.0.smithi110.stdout:412: done (2 left) 2024-09-14T03:38:53.096 INFO:tasks.rados.rados.0.smithi110.stdout:416: read oid 14 snap -1 2024-09-14T03:38:53.096 INFO:tasks.rados.rados.0.smithi110.stdout:416: expect (ObjNum 118 snap 29 seq_num 118) 2024-09-14T03:38:53.096 INFO:tasks.rados.rados.0.smithi110.stdout:417: snap_create 2024-09-14T03:38:53.096 INFO:tasks.rados.rados.0.smithi110.stdout:414: finishing write tid 4 to smithi11075562-38 2024-09-14T03:38:53.097 INFO:tasks.rados.rados.0.smithi110.stdout:414: finishing write tid 5 to smithi11075562-38 2024-09-14T03:38:53.098 INFO:tasks.rados.rados.0.smithi110.stdout:414: finishing write tid 6 to smithi11075562-38 2024-09-14T03:38:53.098 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 66 (ObjNum 142 snap 39 seq_num 142) dirty exists 2024-09-14T03:38:53.098 INFO:tasks.rados.rados.0.smithi110.stdout:414: left oid 38 (ObjNum 142 snap 39 seq_num 142) 2024-09-14T03:38:53.101 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 146 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:38:53.108 INFO:tasks.rados.rados.0.smithi110.stdout:416: expect (ObjNum 118 snap 29 seq_num 118) 2024-09-14T03:38:53.333 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:53 smithi195 ceph-mon[30695]: pgmap v207: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.5 MiB/s wr, 7 op/s 2024-09-14T03:38:53.333 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:53 smithi195 ceph-mon[30695]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:38:53.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:53 smithi110 ceph-mon[35374]: pgmap v207: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.5 MiB/s wr, 7 op/s 2024-09-14T03:38:53.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:53 smithi110 ceph-mon[35374]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:38:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:53 smithi110 ceph-mon[27365]: pgmap v207: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.5 MiB/s wr, 7 op/s 2024-09-14T03:38:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:53 smithi110 ceph-mon[27365]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:38:54.037 INFO:tasks.rados.rados.0.smithi110.stdout:414: done (3 left) 2024-09-14T03:38:54.037 INFO:tasks.rados.rados.0.smithi110.stdout:415: done (2 left) 2024-09-14T03:38:54.037 INFO:tasks.rados.rados.0.smithi110.stdout:416: done (1 left) 2024-09-14T03:38:54.037 INFO:tasks.rados.rados.0.smithi110.stdout:417: done (0 left) 2024-09-14T03:38:54.037 INFO:tasks.rados.rados.0.smithi110.stdout:418: delete oid 27 current snap is 40 2024-09-14T03:38:54.039 INFO:tasks.rados.rados.0.smithi110.stdout:418: done (0 left) 2024-09-14T03:38:54.039 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 40 2024-09-14T03:38:54.040 INFO:tasks.rados.rados.0.smithi110.stdout:419: seq_num 144 ranges {0=16384} 2024-09-14T03:38:54.040 INFO:tasks.rados.rados.0.smithi110.stdout:419: writing smithi11075562-3 from 0 to 16384 tid 1 2024-09-14T03:38:54.040 INFO:tasks.rados.rados.0.smithi110.stdout:420: snap_create 2024-09-14T03:38:54.041 INFO:tasks.rados.rados.0.smithi110.stdout:419: finishing write tid 1 to smithi11075562-3 2024-09-14T03:38:54.041 INFO:tasks.rados.rados.0.smithi110.stdout:419: finishing write tid 2 to smithi11075562-3 2024-09-14T03:38:54.042 INFO:tasks.rados.rados.0.smithi110.stdout:419: finishing write tid 3 to smithi11075562-3 2024-09-14T03:38:54.042 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 97 (ObjNum 144 snap 40 seq_num 144) dirty exists 2024-09-14T03:38:54.043 INFO:tasks.rados.rados.0.smithi110.stdout:419: left oid 3 (ObjNum 144 snap 40 seq_num 144) 2024-09-14T03:38:54.296 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:54 smithi195 ceph-mon[30695]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:38:54.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:54 smithi110 ceph-mon[35374]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:38:54.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:54 smithi110 ceph-mon[27365]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:38:55.024 INFO:tasks.rados.rados.0.smithi110.stdout:419: done (1 left) 2024-09-14T03:38:55.024 INFO:tasks.rados.rados.0.smithi110.stdout:420: done (0 left) 2024-09-14T03:38:55.024 INFO:tasks.rados.rados.0.smithi110.stdout:421: rollback oid 27 current snap is 41 2024-09-14T03:38:55.025 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 40 2024-09-14T03:38:55.025 INFO:tasks.rados.rados.0.smithi110.stdout:422: setattr oid 20 current snap is 41 2024-09-14T03:38:55.025 INFO:tasks.rados.rados.0.smithi110.stdout:423: snap_create 2024-09-14T03:38:55.025 INFO:tasks.rados.rados.0.smithi110.stdout:421: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:38:55.025 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 82 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:38:55.030 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 148 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-14T03:38:55.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:55 smithi195 ceph-mon[30695]: pgmap v210: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.5 MiB/s wr, 7 op/s 2024-09-14T03:38:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:55 smithi195 ceph-mon[30695]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:38:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:55 smithi195 ceph-mon[30695]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:38:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:55 smithi110 ceph-mon[35374]: pgmap v210: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.5 MiB/s wr, 7 op/s 2024-09-14T03:38:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:55 smithi110 ceph-mon[35374]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:38:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:55 smithi110 ceph-mon[35374]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:38:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:55 smithi110 ceph-mon[27365]: pgmap v210: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.5 MiB/s wr, 7 op/s 2024-09-14T03:38:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:55 smithi110 ceph-mon[27365]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:38:55.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:55 smithi110 ceph-mon[27365]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:38:56.025 INFO:tasks.rados.rados.0.smithi110.stdout:421: done (2 left) 2024-09-14T03:38:56.025 INFO:tasks.rados.rados.0.smithi110.stdout:422: done (1 left) 2024-09-14T03:38:56.025 INFO:tasks.rados.rados.0.smithi110.stdout:423: done (0 left) 2024-09-14T03:38:56.025 INFO:tasks.rados.rados.0.smithi110.stdout:424: read oid 46 snap -1 2024-09-14T03:38:56.026 INFO:tasks.rados.rados.0.smithi110.stdout:424: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:38:56.026 INFO:tasks.rados.rados.0.smithi110.stdout:425: read oid 7 snap -1 2024-09-14T03:38:56.026 INFO:tasks.rados.rados.0.smithi110.stdout:425: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:38:56.026 INFO:tasks.rados.rados.0.smithi110.stdout:426: read oid 38 snap 39 2024-09-14T03:38:56.026 INFO:tasks.rados.rados.0.smithi110.stdout:426: expect (ObjNum 142 snap 39 seq_num 142) 2024-09-14T03:38:56.026 INFO:tasks.rados.rados.0.smithi110.stdout:427: snap_remove snap 41 2024-09-14T03:38:56.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[35374]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:38:56.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:56 smithi110 ceph-mon[27365]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:38:56.704 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:56 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.704 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:56 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.704 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:56 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.704 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:56 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:38:56.704 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:56 smithi195 ceph-mon[30695]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:38:57.028 INFO:tasks.rados.rados.0.smithi110.stdout:427: done (3 left) 2024-09-14T03:38:57.028 INFO:tasks.rados.rados.0.smithi110.stdout:428: snap_create 2024-09-14T03:38:57.028 INFO:tasks.rados.rados.0.smithi110.stdout:424: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:38:57.048 INFO:tasks.rados.rados.0.smithi110.stdout:425: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:38:57.050 INFO:tasks.rados.rados.0.smithi110.stdout:426: expect (ObjNum 142 snap 39 seq_num 142) 2024-09-14T03:38:57.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:57 smithi195 ceph-mon[30695]: pgmap v213: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:57.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:57 smithi195 ceph-mon[30695]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:38:57.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:57 smithi110 ceph-mon[35374]: pgmap v213: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:57.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:57 smithi110 ceph-mon[35374]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:38:57.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:57 smithi110 ceph-mon[27365]: pgmap v213: 121 pgs: 121 active+clean; 117 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:38:57.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:57 smithi110 ceph-mon[27365]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:38:58.028 INFO:tasks.rados.rados.0.smithi110.stdout:424: done (3 left) 2024-09-14T03:38:58.028 INFO:tasks.rados.rados.0.smithi110.stdout:425: done (2 left) 2024-09-14T03:38:58.028 INFO:tasks.rados.rados.0.smithi110.stdout:426: done (1 left) 2024-09-14T03:38:58.028 INFO:tasks.rados.rados.0.smithi110.stdout:428: done (0 left) 2024-09-14T03:38:58.028 INFO:tasks.rados.rados.0.smithi110.stdout:429: rmattr oid 6 current snap is 43 2024-09-14T03:38:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:430: rollback oid 3 current snap is 43 2024-09-14T03:38:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 3 to 25 2024-09-14T03:38:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:431: rollback oid 16 current snap is 43 2024-09-14T03:38:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 20 2024-09-14T03:38:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:432: snap_create 2024-09-14T03:38:58.031 INFO:tasks.rados.rados.0.smithi110.stdout:430: finishing rollback tid 0 to smithi11075562-3 2024-09-14T03:38:58.031 INFO:tasks.rados.rados.0.smithi110.stdout:430: finishing rollback tid 1 to smithi11075562-3 2024-09-14T03:38:58.031 INFO:tasks.rados.rados.0.smithi110.stdout:430: finishing rollback tid 2 to smithi11075562-3 2024-09-14T03:38:58.031 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 101 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:38:58.033 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 167 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-14T03:38:58.033 INFO:tasks.rados.rados.0.smithi110.stdout:431: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:38:58.033 INFO:tasks.rados.rados.0.smithi110.stdout:431: finishing rollback tid 2 to smithi11075562-16 2024-09-14T03:38:58.033 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 169 (ObjNum 69 snap 8 seq_num 69) dirty exists 2024-09-14T03:38:59.030 INFO:tasks.rados.rados.0.smithi110.stdout:429: done (3 left) 2024-09-14T03:38:59.030 INFO:tasks.rados.rados.0.smithi110.stdout:430: done (2 left) 2024-09-14T03:38:59.030 INFO:tasks.rados.rados.0.smithi110.stdout:431: done (1 left) 2024-09-14T03:38:59.030 INFO:tasks.rados.rados.0.smithi110.stdout:432: done (0 left) 2024-09-14T03:38:59.030 INFO:tasks.rados.rados.0.smithi110.stdout:433: rollback oid 39 current snap is 44 2024-09-14T03:38:59.030 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 32 2024-09-14T03:38:59.030 INFO:tasks.rados.rados.0.smithi110.stdout:434: rollback oid 21 current snap is 44 2024-09-14T03:38:59.030 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 43 2024-09-14T03:38:59.031 INFO:tasks.rados.rados.0.smithi110.stdout:435: delete oid 24 current snap is 44 2024-09-14T03:38:59.037 INFO:tasks.rados.rados.0.smithi110.stdout:435: done (2 left) 2024-09-14T03:38:59.037 INFO:tasks.rados.rados.0.smithi110.stdout:436: delete oid 3 current snap is 44 2024-09-14T03:38:59.037 INFO:tasks.rados.rados.0.smithi110.stdout:433: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:38:59.038 INFO:tasks.rados.rados.0.smithi110.stdout:433: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:433: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 176 (ObjNum 66 snap 6 seq_num 66) dirty exists 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:433: done (2 left) 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:436: done (1 left) 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:437: read oid 27 snap 42 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:437: expect deleted 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:438: setattr oid 12 current snap is 44 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:434: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:38:59.039 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 151 (ObjNum 65 snap 0 seq_num 64898864) dirty dne 2024-09-14T03:38:59.042 INFO:tasks.rados.rados.0.smithi110.stdout:434: done (2 left) 2024-09-14T03:38:59.042 INFO:tasks.rados.rados.0.smithi110.stdout:437: done (1 left) 2024-09-14T03:38:59.042 INFO:tasks.rados.rados.0.smithi110.stdout:439: setattr oid 30 current snap is 44 2024-09-14T03:38:59.045 INFO:tasks.rados.rados.0.smithi110.stdout:440: delete oid 40 current snap is 44 2024-09-14T03:38:59.045 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 244 (ObjNum 139 snap 38 seq_num 139) dirty exists 2024-09-14T03:38:59.046 INFO:tasks.rados.rados.0.smithi110.stdout:438: done (2 left) 2024-09-14T03:38:59.046 INFO:tasks.rados.rados.0.smithi110.stdout:440: done (1 left) 2024-09-14T03:38:59.046 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 44 2024-09-14T03:38:59.047 INFO:tasks.rados.rados.0.smithi110.stdout:441: seq_num 148 ranges {0=81920} 2024-09-14T03:38:59.047 INFO:tasks.rados.rados.0.smithi110.stdout:441: writing smithi11075562-28 from 0 to 81920 tid 1 2024-09-14T03:38:59.047 INFO:tasks.rados.rados.0.smithi110.stdout:442: snap_remove snap 43 2024-09-14T03:38:59.091 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:59 smithi195 ceph-mon[30695]: pgmap v216: 121 pgs: 121 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.1 MiB/s wr, 27 op/s 2024-09-14T03:38:59.091 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:38:59 smithi195 ceph-mon[30695]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:38:59.220 INFO:tasks.rados.rados.0.smithi110.stdout:442: done (2 left) 2024-09-14T03:38:59.220 INFO:tasks.rados.rados.0.smithi110.stdout:443: copy_from oid 22 from oid 13 current snap is 44 2024-09-14T03:38:59.220 INFO:tasks.rados.rados.0.smithi110.stdout:444: read oid 32 snap -1 2024-09-14T03:38:59.220 INFO:tasks.rados.rados.0.smithi110.stdout:444: expect (ObjNum 109 snap 25 seq_num 109) 2024-09-14T03:38:59.220 INFO:tasks.rados.rados.0.smithi110.stdout:445: rollback oid 15 current snap is 44 2024-09-14T03:38:59.221 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 42 2024-09-14T03:38:59.221 INFO:tasks.rados.rados.0.smithi110.stdout:446: snap_remove snap 32 2024-09-14T03:38:59.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-mon[35374]: pgmap v216: 121 pgs: 121 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.1 MiB/s wr, 27 op/s 2024-09-14T03:38:59.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-mon[35374]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:38:59.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-mon[27365]: pgmap v216: 121 pgs: 121 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.1 MiB/s wr, 27 op/s 2024-09-14T03:38:59.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-mon[27365]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:38:59.360 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ignoring --setuser ceph since I am not root 2024-09-14T03:38:59.360 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ignoring --setgroup ceph since I am not root 2024-09-14T03:38:59.360 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-mgr[27599]: -- 172.21.15.110:0/3237436904 <== mon.0 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560540bf6340 con 0x560540c02c00 2024-09-14T03:38:59.360 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:38:59.357+0000 7f53c8b09ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:38:59.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:38:59] ENGINE Bus STOPPING 2024-09-14T03:38:59.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:38:59.411+0000 7f53c8b09ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:38:59.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:38:59.474+0000 7f53c8b09ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:38:59.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:38:59.594+0000 7f53c8b09ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:38:59.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:38:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:38:59.959 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:38:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:38:59] ENGINE Bus STOPPED 2024-09-14T03:38:59.981 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:38:59.730+0000 7f53c8b09ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:39:00.024 INFO:tasks.rados.rados.0.smithi110.stdout:446: done (5 left) 2024-09-14T03:39:00.024 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 44 2024-09-14T03:39:00.024 INFO:tasks.rados.rados.0.smithi110.stdout:447: seq_num 149 ranges {696320=98304} 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:447: writing smithi11075562-42 from 696320 to 794624 tid 1 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:448: delete oid 14 current snap is 44 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 129 (ObjNum 93 snap 19 seq_num 93) dirty exists 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:441: finishing write tid 1 to smithi11075562-28 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:441: finishing write tid 2 to smithi11075562-28 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:441: finishing write tid 3 to smithi11075562-28 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 105 (ObjNum 148 snap 44 seq_num 148) dirty exists 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:441: left oid 28 (ObjNum 148 snap 44 seq_num 148) 2024-09-14T03:39:00.029 INFO:tasks.rados.rados.0.smithi110.stdout:444: expect (ObjNum 109 snap 25 seq_num 109) 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:445: finishing rollback tid 0 to smithi11075562-15 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:445: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:445: finishing rollback tid 2 to smithi11075562-15 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 133 (ObjNum 126 snap 33 seq_num 126) dirty exists 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:443: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:443: finishing copy_from to smithi11075562-22 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 153 (ObjNum 140 snap 39 seq_num 140) dirty exists 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:439: done (6 left) 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:441: done (5 left) 2024-09-14T03:39:00.110 INFO:tasks.rados.rados.0.smithi110.stdout:443: done (4 left) 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:444: done (3 left) 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:445: done (2 left) 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:448: done (1 left) 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:449: read oid 25 snap 40 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:449: expect (ObjNum 129 snap 34 seq_num 129) 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:450: read oid 47 snap -1 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:450: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:451: read oid 37 snap -1 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:451: expect deleted 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:452: rollback oid 2 current snap is 44 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 2 to 39 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:447: finishing write tid 1 to smithi11075562-42 2024-09-14T03:39:00.111 INFO:tasks.rados.rados.0.smithi110.stdout:447: finishing write tid 2 to smithi11075562-42 2024-09-14T03:39:00.112 INFO:tasks.rados.rados.0.smithi110.stdout:447: finishing write tid 3 to smithi11075562-42 2024-09-14T03:39:00.112 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 158 (ObjNum 149 snap 44 seq_num 149) dirty exists 2024-09-14T03:39:00.112 INFO:tasks.rados.rados.0.smithi110.stdout:447: left oid 42 (ObjNum 149 snap 44 seq_num 149) 2024-09-14T03:39:00.112 INFO:tasks.rados.rados.0.smithi110.stdout:447: done (4 left) 2024-09-14T03:39:00.112 INFO:tasks.rados.rados.0.smithi110.stdout:453: snap_create 2024-09-14T03:39:00.112 INFO:tasks.rados.rados.0.smithi110.stdout:452: finishing rollback tid 0 to smithi11075562-2 2024-09-14T03:39:00.112 INFO:tasks.rados.rados.0.smithi110.stdout:452: finishing rollback tid 1 to smithi11075562-2 2024-09-14T03:39:00.113 INFO:tasks.rados.rados.0.smithi110.stdout:452: finishing rollback tid 2 to smithi11075562-2 2024-09-14T03:39:00.113 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 115 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:39:00.129 INFO:tasks.rados.rados.0.smithi110.stdout:449: expect (ObjNum 129 snap 34 seq_num 129) 2024-09-14T03:39:00.294 INFO:tasks.rados.rados.0.smithi110.stdout:450: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:39:00.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:39:00.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.394 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: Activating manager daemon x 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.14526 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: mgrmap e24: x(active, starting, since 0.00689373s) 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:39:00.396 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:39:00.397 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:39:00.397 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: Manager daemon x is now available 2024-09-14T03:39:00.397 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-mon[30695]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:39:00.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:38:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:38:59.980+0000 7f53c8b09ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:39:00.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:00.048+0000 7f53c8b09ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:39:00.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:00.222+0000 7f53c8b09ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:39:00.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: Activating manager daemon x 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.14526 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:39:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: mgrmap e24: x(active, starting, since 0.00689373s) 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:00.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:00.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 172.21.15.110:0/3054463954' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: Activating manager daemon x 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.14526 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: mgrmap e24: x(active, starting, since 0.00689373s) 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:39:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:39:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: Manager daemon x is now available 2024-09-14T03:39:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[35374]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:39:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: Manager daemon x is now available 2024-09-14T03:39:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:00 smithi110 ceph-mon[27365]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:39:00.647 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:39:00] ENGINE Bus STARTING 2024-09-14T03:39:00.647 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.446+0000 7feca7b47640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.648 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.446+0000 7feca7b47640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.648 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.446+0000 7feca7b47640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.648 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.446+0000 7feca7b47640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.649 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.446+0000 7feca7b47640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.649 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.447+0000 7feca9b4b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.649 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.447+0000 7feca9b4b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.649 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.447+0000 7feca9b4b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.649 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.447+0000 7feca9b4b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.649 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:00.447+0000 7feca9b4b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:39:00.649 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:39:00] ENGINE Serving on http://:::9283 2024-09-14T03:39:00.649 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:39:00] ENGINE Bus STARTED 2024-09-14T03:39:01.037 INFO:tasks.rados.rados.0.smithi110.stdout:449: done (4 left) 2024-09-14T03:39:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:450: done (3 left) 2024-09-14T03:39:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:451: done (2 left) 2024-09-14T03:39:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:452: done (1 left) 2024-09-14T03:39:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:453: done (0 left) 2024-09-14T03:39:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:454: read oid 14 snap -1 2024-09-14T03:39:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:454: expect deleted 2024-09-14T03:39:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:455: delete oid 12 current snap is 45 2024-09-14T03:39:01.043 INFO:tasks.rados.rados.0.smithi110.stdout:454: done (1 left) 2024-09-14T03:39:01.043 INFO:tasks.rados.rados.0.smithi110.stdout:455: done (0 left) 2024-09-14T03:39:01.043 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 45 2024-09-14T03:39:01.043 INFO:tasks.rados.rados.0.smithi110.stdout:456: seq_num 152 ranges {0=147456} 2024-09-14T03:39:01.044 INFO:tasks.rados.rados.0.smithi110.stdout:456: writing smithi11075562-3 from 0 to 147456 tid 1 2024-09-14T03:39:01.044 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 45 2024-09-14T03:39:01.044 INFO:tasks.rados.rados.0.smithi110.stdout:457: seq_num 153 ranges {3932160=163840} 2024-09-14T03:39:01.067 INFO:tasks.rados.rados.0.smithi110.stdout:457: writing smithi11075562-49 from 3932160 to 4096000 tid 1 2024-09-14T03:39:01.067 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 45 2024-09-14T03:39:01.067 INFO:tasks.rados.rados.0.smithi110.stdout:456: finishing write tid 1 to smithi11075562-3 2024-09-14T03:39:01.067 INFO:tasks.rados.rados.0.smithi110.stdout:456: finishing write tid 2 to smithi11075562-3 2024-09-14T03:39:01.067 INFO:tasks.rados.rados.0.smithi110.stdout:456: finishing write tid 3 to smithi11075562-3 2024-09-14T03:39:01.068 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 117 (ObjNum 152 snap 45 seq_num 152) dirty exists 2024-09-14T03:39:01.068 INFO:tasks.rados.rados.0.smithi110.stdout:456: left oid 3 (ObjNum 152 snap 45 seq_num 152) 2024-09-14T03:39:01.068 INFO:tasks.rados.rados.0.smithi110.stdout:458: seq_num 154 ranges {106496=180224} 2024-09-14T03:39:01.069 INFO:tasks.rados.rados.0.smithi110.stdout:458: writing smithi11075562-7 from 106496 to 286720 tid 1 2024-09-14T03:39:01.069 INFO:tasks.rados.rados.0.smithi110.stdout:456: done (2 left) 2024-09-14T03:39:01.069 INFO:tasks.rados.rados.0.smithi110.stdout:459: snap_remove snap 40 2024-09-14T03:39:01.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:01.133+0000 7f53c8b09ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:39:01.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:01.201+0000 7f53c8b09ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:39:01.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: mgrmap e25: x(active, since 1.00875s) 2024-09-14T03:39:01.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: Checking for cert/key for grafana.a 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: Migrating grafana.a cert to cert store 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: Migrating grafana.a key to cert store 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:39:01.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[35374]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: mgrmap e25: x(active, since 1.00875s) 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: Checking for cert/key for grafana.a 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: Migrating grafana.a cert to cert store 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: Migrating grafana.a key to cert store 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:39:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-mon[27365]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:39:01.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: mgrmap e25: x(active, since 1.00875s) 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: Checking for cert/key for grafana.a 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: Migrating grafana.a cert to cert store 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: Migrating grafana.a key to cert store 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:39:01.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-mon[30695]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:01.258+0000 7fec940e3640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:39:01] ENGINE Error in HTTPServer.serve 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: Traceback (most recent call last): 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: self._connections.run(self.expiration_interval) 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:39:01.711 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: self._run(expiration_interval) 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: s = self.context.wrap_socket( 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: return self.sslsocket_class._create( 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: self.do_handshake() 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: self._sslobj.do_handshake() 2024-09-14T03:39:01.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:39:01.713 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:39:02.034 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:01.922+0000 7f53c8b09ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:39:02.034 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.032+0000 7f53c8b09ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:457: finishing write tid 1 to smithi11075562-49 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:457: finishing write tid 2 to smithi11075562-49 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:457: finishing write tid 3 to smithi11075562-49 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 161 (ObjNum 153 snap 45 seq_num 153) dirty exists 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:457: left oid 49 (ObjNum 153 snap 45 seq_num 153) 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:458: finishing write tid 1 to smithi11075562-7 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:458: finishing write tid 2 to smithi11075562-7 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:458: finishing write tid 3 to smithi11075562-7 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 71 (ObjNum 154 snap 45 seq_num 154) dirty exists 2024-09-14T03:39:02.038 INFO:tasks.rados.rados.0.smithi110.stdout:458: left oid 7 (ObjNum 154 snap 45 seq_num 154) 2024-09-14T03:39:02.039 INFO:tasks.rados.rados.0.smithi110.stdout:457: done (2 left) 2024-09-14T03:39:02.039 INFO:tasks.rados.rados.0.smithi110.stdout:458: done (1 left) 2024-09-14T03:39:02.039 INFO:tasks.rados.rados.0.smithi110.stdout:459: done (0 left) 2024-09-14T03:39:02.039 INFO:tasks.rados.rados.0.smithi110.stdout:460: rollback oid 24 current snap is 45 2024-09-14T03:39:02.039 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 24 2024-09-14T03:39:02.039 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 45 2024-09-14T03:39:02.039 INFO:tasks.rados.rados.0.smithi110.stdout:461: seq_num 155 ranges {0=204800} 2024-09-14T03:39:02.040 INFO:tasks.rados.rados.0.smithi110.stdout:461: writing smithi11075562-14 from 0 to 204800 tid 1 2024-09-14T03:39:02.040 INFO:tasks.rados.rados.0.smithi110.stdout:462: copy_from oid 6 from oid 19 current snap is 45 2024-09-14T03:39:02.040 INFO:tasks.rados.rados.0.smithi110.stdout:463: snap_create 2024-09-14T03:39:02.045 INFO:tasks.rados.rados.0.smithi110.stdout:461: finishing write tid 1 to smithi11075562-14 2024-09-14T03:39:02.045 INFO:tasks.rados.rados.0.smithi110.stdout:461: finishing write tid 2 to smithi11075562-14 2024-09-14T03:39:02.047 INFO:tasks.rados.rados.0.smithi110.stdout:460: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:39:02.047 INFO:tasks.rados.rados.0.smithi110.stdout:460: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:39:02.047 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 85 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-14T03:39:02.048 INFO:tasks.rados.rados.0.smithi110.stdout:461: finishing write tid 3 to smithi11075562-14 2024-09-14T03:39:02.048 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 189 (ObjNum 155 snap 45 seq_num 155) dirty exists 2024-09-14T03:39:02.048 INFO:tasks.rados.rados.0.smithi110.stdout:461: left oid 14 (ObjNum 155 snap 45 seq_num 155) 2024-09-14T03:39:02.055 INFO:tasks.rados.rados.0.smithi110.stdout:462: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:39:02.056 INFO:tasks.rados.rados.0.smithi110.stdout:462: finishing copy_from to smithi11075562-6 2024-09-14T03:39:02.057 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 119 (ObjNum 126 snap 33 seq_num 126) dirty exists 2024-09-14T03:39:02.348 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.161+0000 7f53c8b09ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:39:02.348 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.221+0000 7f53c8b09ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:39:02.348 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.346+0000 7f53c8b09ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: Deploying cephadm binary to smithi195 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: [14/Sep/2024:03:39:01] ENGINE Bus STARTING 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: pgmap v6: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: [14/Sep/2024:03:39:01] ENGINE Serving on https://172.21.15.195:7150 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: [14/Sep/2024:03:39:01] ENGINE Error in HTTPServer.serve 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout: Traceback (most recent call last): 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:39:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._run(expiration_interval) 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: s = self.context.wrap_socket( 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: return self.sslsocket_class._create( 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: self.do_handshake() 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._sslobj.do_handshake() 2024-09-14T03:39:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:39:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: [14/Sep/2024:03:39:01] ENGINE Serving on http://172.21.15.195:8765 2024-09-14T03:39:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: [14/Sep/2024:03:39:01] ENGINE Bus STARTED 2024-09-14T03:39:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: mgrmap e26: x(active, since 2s) 2024-09-14T03:39:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:02 smithi195 ceph-mon[30695]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:39:02.617 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.417+0000 7f53c8b09ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:39:02.618 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.477+0000 7f53c8b09ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:39:02.618 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.616+0000 7f53c8b09ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:39:02.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: Deploying cephadm binary to smithi195 2024-09-14T03:39:02.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: [14/Sep/2024:03:39:01] ENGINE Bus STARTING 2024-09-14T03:39:02.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: pgmap v6: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:02.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: [14/Sep/2024:03:39:01] ENGINE Serving on https://172.21.15.195:7150 2024-09-14T03:39:02.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: [14/Sep/2024:03:39:01] ENGINE Error in HTTPServer.serve 2024-09-14T03:39:02.619 INFO:journalctl@ceph.mon.a.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:39:02.619 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:39:02.619 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:39:02.619 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:39:02.619 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:39:02.619 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:39:02.619 INFO:journalctl@ceph.mon.a.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:39:02.620 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:39:02.620 INFO:journalctl@ceph.mon.a.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:39:02.620 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:39:02.620 INFO:journalctl@ceph.mon.a.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:39:02.620 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:39:02.620 INFO:journalctl@ceph.mon.a.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:39:02.620 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:39:02.620 INFO:journalctl@ceph.mon.a.smithi110.stdout: self.do_handshake() 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.a.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: [14/Sep/2024:03:39:01] ENGINE Serving on http://172.21.15.195:8765 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: [14/Sep/2024:03:39:01] ENGINE Bus STARTED 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: mgrmap e26: x(active, since 2s) 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[27365]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: Deploying cephadm binary to smithi195 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: [14/Sep/2024:03:39:01] ENGINE Bus STARTING 2024-09-14T03:39:02.621 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: pgmap v6: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: [14/Sep/2024:03:39:01] ENGINE Serving on https://172.21.15.195:7150 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: [14/Sep/2024:03:39:01] ENGINE Error in HTTPServer.serve 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:39:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: self.do_handshake() 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: [14/Sep/2024:03:39:01] ENGINE Serving on http://172.21.15.195:8765 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: [14/Sep/2024:03:39:01] ENGINE Bus STARTED 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: mgrmap e26: x(active, since 2s) 2024-09-14T03:39:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-mon[35374]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:39:02.907 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.689+0000 7f53c8b09ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:39:02.907 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.764+0000 7f53c8b09ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:39:02.907 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.846+0000 7f53c8b09ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:39:03.041 INFO:tasks.rados.rados.0.smithi110.stdout:460: done (3 left) 2024-09-14T03:39:03.041 INFO:tasks.rados.rados.0.smithi110.stdout:461: done (2 left) 2024-09-14T03:39:03.041 INFO:tasks.rados.rados.0.smithi110.stdout:462: done (1 left) 2024-09-14T03:39:03.041 INFO:tasks.rados.rados.0.smithi110.stdout:463: done (0 left) 2024-09-14T03:39:03.041 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 46 2024-09-14T03:39:03.041 INFO:tasks.rados.rados.0.smithi110.stdout:464: seq_num 156 ranges {0=221184} 2024-09-14T03:39:03.042 INFO:tasks.rados.rados.0.smithi110.stdout:464: writing smithi11075562-12 from 0 to 221184 tid 1 2024-09-14T03:39:03.042 INFO:tasks.rados.rados.0.smithi110.stdout:465: read oid 9 snap -1 2024-09-14T03:39:03.042 INFO:tasks.rados.rados.0.smithi110.stdout:465: expect (ObjNum 104 snap 20 seq_num 104) 2024-09-14T03:39:03.042 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 46 2024-09-14T03:39:03.042 INFO:tasks.rados.rados.0.smithi110.stdout:466: seq_num 157 ranges {606208=237568} 2024-09-14T03:39:03.046 INFO:tasks.rados.rados.0.smithi110.stdout:466: writing smithi11075562-26 from 606208 to 843776 tid 1 2024-09-14T03:39:03.046 INFO:tasks.rados.rados.0.smithi110.stdout:467: snap_create 2024-09-14T03:39:03.049 INFO:tasks.rados.rados.0.smithi110.stdout:464: finishing write tid 1 to smithi11075562-12 2024-09-14T03:39:03.050 INFO:tasks.rados.rados.0.smithi110.stdout:464: finishing write tid 2 to smithi11075562-12 2024-09-14T03:39:03.051 INFO:tasks.rados.rados.0.smithi110.stdout:464: finishing write tid 3 to smithi11075562-12 2024-09-14T03:39:03.051 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 260 (ObjNum 156 snap 46 seq_num 156) dirty exists 2024-09-14T03:39:03.052 INFO:tasks.rados.rados.0.smithi110.stdout:464: left oid 12 (ObjNum 156 snap 46 seq_num 156) 2024-09-14T03:39:03.052 INFO:tasks.rados.rados.0.smithi110.stdout:466: finishing write tid 1 to smithi11075562-26 2024-09-14T03:39:03.053 INFO:tasks.rados.rados.0.smithi110.stdout:466: finishing write tid 2 to smithi11075562-26 2024-09-14T03:39:03.053 INFO:tasks.rados.rados.0.smithi110.stdout:466: finishing write tid 3 to smithi11075562-26 2024-09-14T03:39:03.053 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 24 (ObjNum 157 snap 46 seq_num 157) dirty exists 2024-09-14T03:39:03.053 INFO:tasks.rados.rados.0.smithi110.stdout:466: left oid 26 (ObjNum 157 snap 46 seq_num 157) 2024-09-14T03:39:03.062 INFO:tasks.rados.rados.0.smithi110.stdout:465: expect (ObjNum 104 snap 20 seq_num 104) 2024-09-14T03:39:03.181 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.905+0000 7f53c8b09ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:39:03.181 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:02.964+0000 7f53c8b09ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:39:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-mon[27365]: Deploying cephadm binary to smithi110 2024-09-14T03:39:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-mon[27365]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:39:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-mon[35374]: Deploying cephadm binary to smithi110 2024-09-14T03:39:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-mon[35374]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:39:03.701 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:03 smithi195 ceph-mon[30695]: Deploying cephadm binary to smithi110 2024-09-14T03:39:03.701 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:03 smithi195 ceph-mon[30695]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:39:04.042 INFO:tasks.rados.rados.0.smithi110.stdout:464: done (3 left) 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:465: done (2 left) 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:466: done (1 left) 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:467: done (0 left) 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:468: copy_from oid 15 from oid 40 current snap is 47 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:469: read oid 7 snap 25 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:469: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:470: rollback oid 31 current snap is 47 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 42 2024-09-14T03:39:04.043 INFO:tasks.rados.rados.0.smithi110.stdout:471: delete oid 35 current snap is 47 2024-09-14T03:39:04.045 INFO:tasks.rados.rados.0.smithi110.stdout:468: finishing copy_from to smithi11075562-15 2024-09-14T03:39:04.045 INFO:tasks.rados.rados.0.smithi110.stdout:468: got expected ENOENT (src dne) 2024-09-14T03:39:04.045 INFO:tasks.rados.rados.0.smithi110.stdout:468: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:39:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:468: done (3 left) 2024-09-14T03:39:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:471: done (2 left) 2024-09-14T03:39:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:472: read oid 25 snap -1 2024-09-14T03:39:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:472: expect (ObjNum 129 snap 34 seq_num 129) 2024-09-14T03:39:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:473: delete oid 5 current snap is 47 2024-09-14T03:39:04.046 INFO:tasks.rados.rados.0.smithi110.stdout:469: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:39:04.049 INFO:tasks.rados.rados.0.smithi110.stdout:470: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:39:04.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:39:04.066 INFO:tasks.rados.rados.0.smithi110.stdout:472: expect (ObjNum 129 snap 34 seq_num 129) 2024-09-14T03:39:04.130 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:03.790+0000 7f53c8b09ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:39:04.130 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:39:03] ENGINE Bus STARTING 2024-09-14T03:39:04.130 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: CherryPy Checker: 2024-09-14T03:39:04.130 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: The Application mounted at '' has an empty config. 2024-09-14T03:39:04.130 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: 2024-09-14T03:39:04.130 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:39:03] ENGINE Serving on http://:::9283 2024-09-14T03:39:04.131 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: [14/Sep/2024:03:39:03] ENGINE Bus STARTED 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:469: done (3 left) 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:470: done (2 left) 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:472: done (1 left) 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:473: done (0 left) 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:474: rmattr oid 9 current snap is 47 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:474: done (0 left) 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:475: copy_from oid 40 from oid 24 current snap is 47 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 47 2024-09-14T03:39:04.228 INFO:tasks.rados.rados.0.smithi110.stdout:476: seq_num 160 ranges {4096000=286720} 2024-09-14T03:39:04.252 INFO:tasks.rados.rados.0.smithi110.stdout:476: writing smithi11075562-49 from 4096000 to 4382720 tid 1 2024-09-14T03:39:04.252 INFO:tasks.rados.rados.0.smithi110.stdout:477: rollback oid 1 current snap is 47 2024-09-14T03:39:04.252 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 1 to 45 2024-09-14T03:39:04.252 INFO:tasks.rados.rados.0.smithi110.stdout:475: finishing copy_from racing read to smithi11075562-40 2024-09-14T03:39:04.253 INFO:tasks.rados.rados.0.smithi110.stdout:475: finishing copy_from to smithi11075562-40 2024-09-14T03:39:04.253 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 85 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-14T03:39:04.253 INFO:tasks.rados.rados.0.smithi110.stdout:475: done (2 left) 2024-09-14T03:39:04.253 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 47 2024-09-14T03:39:04.253 INFO:tasks.rados.rados.0.smithi110.stdout:478: seq_num 161 ranges {0=303104} 2024-09-14T03:39:04.254 INFO:tasks.rados.rados.0.smithi110.stdout:478: writing smithi11075562-41 from 0 to 303104 tid 1 2024-09-14T03:39:04.254 INFO:tasks.rados.rados.0.smithi110.stdout:479: copy_from oid 14 from oid 4 current snap is 47 2024-09-14T03:39:04.254 INFO:tasks.rados.rados.0.smithi110.stdout:480: read oid 22 snap -1 2024-09-14T03:39:04.254 INFO:tasks.rados.rados.0.smithi110.stdout:480: expect (ObjNum 140 snap 39 seq_num 140) 2024-09-14T03:39:04.254 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 47 2024-09-14T03:39:04.254 INFO:tasks.rados.rados.0.smithi110.stdout:481: seq_num 162 ranges {0=319488} 2024-09-14T03:39:04.255 INFO:tasks.rados.rados.0.smithi110.stdout:481: writing smithi11075562-44 from 0 to 319488 tid 1 2024-09-14T03:39:04.256 INFO:tasks.rados.rados.0.smithi110.stdout:482: read oid 6 snap -1 2024-09-14T03:39:04.256 INFO:tasks.rados.rados.0.smithi110.stdout:482: expect (ObjNum 126 snap 33 seq_num 126) 2024-09-14T03:39:04.256 INFO:tasks.rados.rados.0.smithi110.stdout:483: rmattr oid 23 current snap is 47 2024-09-14T03:39:04.256 INFO:tasks.rados.rados.0.smithi110.stdout:483: done (7 left) 2024-09-14T03:39:04.256 INFO:tasks.rados.rados.0.smithi110.stdout:484: setattr oid 23 current snap is 47 2024-09-14T03:39:04.256 INFO:tasks.rados.rados.0.smithi110.stdout:476: finishing write tid 1 to smithi11075562-49 2024-09-14T03:39:04.256 INFO:tasks.rados.rados.0.smithi110.stdout:476: finishing write tid 2 to smithi11075562-49 2024-09-14T03:39:04.256 INFO:tasks.rados.rados.0.smithi110.stdout:477: finishing rollback tid 0 to smithi11075562-1 2024-09-14T03:39:04.258 INFO:tasks.rados.rados.0.smithi110.stdout:476: finishing write tid 3 to smithi11075562-49 2024-09-14T03:39:04.258 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 170 (ObjNum 160 snap 47 seq_num 160) dirty exists 2024-09-14T03:39:04.258 INFO:tasks.rados.rados.0.smithi110.stdout:476: left oid 49 (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:39:04.258 INFO:tasks.rados.rados.0.smithi110.stdout:478: finishing write tid 1 to smithi11075562-41 2024-09-14T03:39:04.258 INFO:tasks.rados.rados.0.smithi110.stdout:476: done (7 left) 2024-09-14T03:39:04.258 INFO:tasks.rados.rados.0.smithi110.stdout:485: rmattr oid 43 current snap is 47 2024-09-14T03:39:04.259 INFO:tasks.rados.rados.0.smithi110.stdout:486: setattr oid 31 current snap is 47 2024-09-14T03:39:04.259 INFO:tasks.rados.rados.0.smithi110.stdout:477: finishing rollback tid 1 to smithi11075562-1 2024-09-14T03:39:04.259 INFO:tasks.rados.rados.0.smithi110.stdout:481: finishing write tid 1 to smithi11075562-44 2024-09-14T03:39:04.260 INFO:tasks.rados.rados.0.smithi110.stdout:477: finishing rollback tid 2 to smithi11075562-1 2024-09-14T03:39:04.260 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 174 (ObjNum 115 snap 28 seq_num 115) dirty exists 2024-09-14T03:39:04.260 INFO:tasks.rados.rados.0.smithi110.stdout:481: finishing write tid 2 to smithi11075562-44 2024-09-14T03:39:04.261 INFO:tasks.rados.rados.0.smithi110.stdout:478: finishing write tid 2 to smithi11075562-41 2024-09-14T03:39:04.261 INFO:tasks.rados.rados.0.smithi110.stdout:477: done (8 left) 2024-09-14T03:39:04.261 INFO:tasks.rados.rados.0.smithi110.stdout:487: snap_create 2024-09-14T03:39:04.263 INFO:tasks.rados.rados.0.smithi110.stdout:479: finishing copy_from racing read to smithi11075562-14 2024-09-14T03:39:04.263 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 198 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:39:04.264 INFO:tasks.rados.rados.0.smithi110.stdout:481: finishing write tid 3 to smithi11075562-44 2024-09-14T03:39:04.264 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 13 (ObjNum 162 snap 47 seq_num 162) dirty exists 2024-09-14T03:39:04.264 INFO:tasks.rados.rados.0.smithi110.stdout:481: left oid 44 (ObjNum 162 snap 47 seq_num 162) 2024-09-14T03:39:04.264 INFO:tasks.rados.rados.0.smithi110.stdout:479: finishing copy_from to smithi11075562-14 2024-09-14T03:39:04.264 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:39:04.265 INFO:tasks.rados.rados.0.smithi110.stdout:478: finishing write tid 3 to smithi11075562-41 2024-09-14T03:39:04.265 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 196 (ObjNum 161 snap 47 seq_num 161) dirty exists 2024-09-14T03:39:04.265 INFO:tasks.rados.rados.0.smithi110.stdout:478: left oid 41 (ObjNum 161 snap 47 seq_num 161) 2024-09-14T03:39:04.265 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 89 (ObjNum 109 snap 25 seq_num 109) dirty exists 2024-09-14T03:39:04.267 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 57 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:39:04.268 INFO:tasks.rados.rados.0.smithi110.stdout:480: expect (ObjNum 140 snap 39 seq_num 140) 2024-09-14T03:39:04.404 INFO:tasks.rados.rados.0.smithi110.stdout:482: expect (ObjNum 126 snap 33 seq_num 126) 2024-09-14T03:39:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[35374]: pgmap v9: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[35374]: mgrmap e27: x(active, since 4s) 2024-09-14T03:39:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[35374]: Standby manager daemon y started 2024-09-14T03:39:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:39:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:39:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:39:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:39:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[35374]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:39:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[27365]: pgmap v9: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[27365]: mgrmap e27: x(active, since 4s) 2024-09-14T03:39:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[27365]: Standby manager daemon y started 2024-09-14T03:39:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:39:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:39:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:39:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:39:04.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:04 smithi110 ceph-mon[27365]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:39:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:04 smithi195 ceph-mon[30695]: pgmap v9: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:04 smithi195 ceph-mon[30695]: mgrmap e27: x(active, since 4s) 2024-09-14T03:39:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:04 smithi195 ceph-mon[30695]: Standby manager daemon y started 2024-09-14T03:39:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:04 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:39:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:04 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:39:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:04 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:39:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:04 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.110:0/4103659655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:39:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:04 smithi195 ceph-mon[30695]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:478: done (8 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:479: done (7 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:480: done (6 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:481: done (5 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:482: done (4 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:484: done (3 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:485: done (2 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:486: done (1 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:487: done (0 left) 2024-09-14T03:39:05.028 INFO:tasks.rados.rados.0.smithi110.stdout:488: setattr oid 39 current snap is 48 2024-09-14T03:39:05.031 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 48 2024-09-14T03:39:05.031 INFO:tasks.rados.rados.0.smithi110.stdout:489: seq_num 163 ranges {3915776=335872} 2024-09-14T03:39:05.055 INFO:tasks.rados.rados.0.smithi110.stdout:489: writing smithi11075562-22 from 3915776 to 4251648 tid 1 2024-09-14T03:39:05.056 INFO:tasks.rados.rados.0.smithi110.stdout:490: snap_remove snap 38 2024-09-14T03:39:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[35374]: mgrmap e28: x(active, since 5s), standbys: y 2024-09-14T03:39:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:39:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[35374]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:39:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:05.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[27365]: mgrmap e28: x(active, since 5s), standbys: y 2024-09-14T03:39:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:39:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[27365]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:39:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:05 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:05.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:05 smithi195 ceph-mon[30695]: mgrmap e28: x(active, since 5s), standbys: y 2024-09-14T03:39:05.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:05 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:39:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:05 smithi195 ceph-mon[30695]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:39:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:05 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:05 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.034 INFO:tasks.rados.rados.0.smithi110.stdout:490: done (2 left) 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:491: rmattr oid 7 current snap is 48 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:491: done (2 left) 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:492: copy_from oid 19 from oid 37 current snap is 48 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 202 (ObjNum 66 snap 6 seq_num 66) dirty exists 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:489: finishing write tid 1 to smithi11075562-22 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:489: finishing write tid 2 to smithi11075562-22 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:489: finishing write tid 3 to smithi11075562-22 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 155 (ObjNum 163 snap 48 seq_num 163) dirty exists 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:489: left oid 22 (ObjNum 163 snap 48 seq_num 163) 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:488: done (2 left) 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:489: done (1 left) 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:493: copy_from oid 48 from oid 31 current snap is 48 2024-09-14T03:39:06.035 INFO:tasks.rados.rados.0.smithi110.stdout:494: delete oid 16 current snap is 48 2024-09-14T03:39:06.041 INFO:tasks.rados.rados.0.smithi110.stdout:492: finishing copy_from to smithi11075562-19 2024-09-14T03:39:06.041 INFO:tasks.rados.rados.0.smithi110.stdout:492: got expected ENOENT (src dne) 2024-09-14T03:39:06.041 INFO:tasks.rados.rados.0.smithi110.stdout:492: finishing copy_from racing read to smithi11075562-19 2024-09-14T03:39:06.042 INFO:tasks.rados.rados.0.smithi110.stdout:492: done (2 left) 2024-09-14T03:39:06.042 INFO:tasks.rados.rados.0.smithi110.stdout:494: done (1 left) 2024-09-14T03:39:06.042 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 48 2024-09-14T03:39:06.042 INFO:tasks.rados.rados.0.smithi110.stdout:495: seq_num 165 ranges {491520=368640} 2024-09-14T03:39:06.046 INFO:tasks.rados.rados.0.smithi110.stdout:495: writing smithi11075562-43 from 491520 to 860160 tid 1 2024-09-14T03:39:06.047 INFO:tasks.rados.rados.0.smithi110.stdout:496: delete oid 18 current snap is 48 2024-09-14T03:39:06.047 INFO:tasks.rados.rados.0.smithi110.stdout:493: finishing copy_from racing read to smithi11075562-48 2024-09-14T03:39:06.047 INFO:tasks.rados.rados.0.smithi110.stdout:493: finishing copy_from to smithi11075562-48 2024-09-14T03:39:06.047 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 57 (ObjNum 33 snap 0 seq_num 65116464) dirty exists 2024-09-14T03:39:06.049 INFO:tasks.rados.rados.0.smithi110.stdout:493: done (2 left) 2024-09-14T03:39:06.049 INFO:tasks.rados.rados.0.smithi110.stdout:496: done (1 left) 2024-09-14T03:39:06.049 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 48 2024-09-14T03:39:06.049 INFO:tasks.rados.rados.0.smithi110.stdout:497: seq_num 167 ranges {3538944=401408} 2024-09-14T03:39:06.070 INFO:tasks.rados.rados.0.smithi110.stdout:497: writing smithi11075562-15 from 3538944 to 3940352 tid 1 2024-09-14T03:39:06.070 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 48 2024-09-14T03:39:06.070 INFO:tasks.rados.rados.0.smithi110.stdout:498: seq_num 168 ranges {81920=417792} 2024-09-14T03:39:06.073 INFO:tasks.rados.rados.0.smithi110.stdout:498: writing smithi11075562-28 from 81920 to 499712 tid 1 2024-09-14T03:39:06.073 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 48 2024-09-14T03:39:06.073 INFO:tasks.rados.rados.0.smithi110.stdout:495: finishing write tid 1 to smithi11075562-43 2024-09-14T03:39:06.073 INFO:tasks.rados.rados.0.smithi110.stdout:499: seq_num 169 ranges {147456=434176} 2024-09-14T03:39:06.077 INFO:tasks.rados.rados.0.smithi110.stdout:499: writing smithi11075562-3 from 147456 to 581632 tid 1 2024-09-14T03:39:06.077 INFO:tasks.rados.rados.0.smithi110.stdout:500: read oid 40 snap -1 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:500: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 48 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:495: finishing write tid 2 to smithi11075562-43 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:495: finishing write tid 3 to smithi11075562-43 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 213 (ObjNum 165 snap 48 seq_num 165) dirty exists 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:495: left oid 43 (ObjNum 165 snap 48 seq_num 165) 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:497: finishing write tid 1 to smithi11075562-15 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:497: finishing write tid 2 to smithi11075562-15 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:497: finishing write tid 3 to smithi11075562-15 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 161 (ObjNum 167 snap 48 seq_num 167) dirty exists 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:497: left oid 15 (ObjNum 167 snap 48 seq_num 167) 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:498: finishing write tid 1 to smithi11075562-28 2024-09-14T03:39:06.078 INFO:tasks.rados.rados.0.smithi110.stdout:501: seq_num 170 ranges {0=450560} 2024-09-14T03:39:06.080 INFO:tasks.rados.rados.0.smithi110.stdout:501: writing smithi11075562-27 from 0 to 450560 tid 1 2024-09-14T03:39:06.080 INFO:tasks.rados.rados.0.smithi110.stdout:495: done (5 left) 2024-09-14T03:39:06.080 INFO:tasks.rados.rados.0.smithi110.stdout:497: done (4 left) 2024-09-14T03:39:06.081 INFO:tasks.rados.rados.0.smithi110.stdout:502: read oid 14 snap 24 2024-09-14T03:39:06.081 INFO:tasks.rados.rados.0.smithi110.stdout:502: expect (ObjNum 94 snap 19 seq_num 94) 2024-09-14T03:39:06.081 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 48 2024-09-14T03:39:06.081 INFO:tasks.rados.rados.0.smithi110.stdout:498: finishing write tid 2 to smithi11075562-28 2024-09-14T03:39:06.081 INFO:tasks.rados.rados.0.smithi110.stdout:503: seq_num 171 ranges {778240=466944} 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:503: writing smithi11075562-47 from 778240 to 1245184 tid 1 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:504: delete oid 45 current snap is 48 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:498: finishing write tid 3 to smithi11075562-28 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 126 (ObjNum 168 snap 48 seq_num 168) dirty exists 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:498: left oid 28 (ObjNum 168 snap 48 seq_num 168) 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:499: finishing write tid 1 to smithi11075562-3 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:499: finishing write tid 2 to smithi11075562-3 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:499: finishing write tid 3 to smithi11075562-3 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 129 (ObjNum 169 snap 48 seq_num 169) dirty exists 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:499: left oid 3 (ObjNum 169 snap 48 seq_num 169) 2024-09-14T03:39:06.088 INFO:tasks.rados.rados.0.smithi110.stdout:501: finishing write tid 1 to smithi11075562-27 2024-09-14T03:39:06.089 INFO:tasks.rados.rados.0.smithi110.stdout:501: finishing write tid 2 to smithi11075562-27 2024-09-14T03:39:06.089 INFO:tasks.rados.rados.0.smithi110.stdout:501: finishing write tid 3 to smithi11075562-27 2024-09-14T03:39:06.089 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 84 (ObjNum 170 snap 48 seq_num 170) dirty exists 2024-09-14T03:39:06.089 INFO:tasks.rados.rados.0.smithi110.stdout:501: left oid 27 (ObjNum 170 snap 48 seq_num 170) 2024-09-14T03:39:06.089 INFO:tasks.rados.rados.0.smithi110.stdout:500: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:39:06.129 INFO:tasks.rados.rados.0.smithi110.stdout:502: expect (ObjNum 94 snap 19 seq_num 94) 2024-09-14T03:39:06.178 INFO:tasks.rados.rados.0.smithi110.stdout:503: finishing write tid 1 to smithi11075562-47 2024-09-14T03:39:06.178 INFO:tasks.rados.rados.0.smithi110.stdout:503: finishing write tid 2 to smithi11075562-47 2024-09-14T03:39:06.178 INFO:tasks.rados.rados.0.smithi110.stdout:503: finishing write tid 3 to smithi11075562-47 2024-09-14T03:39:06.178 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 266 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:39:06.178 INFO:tasks.rados.rados.0.smithi110.stdout:503: left oid 47 (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:498: done (6 left) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:499: done (5 left) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:500: done (4 left) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:501: done (3 left) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:502: done (2 left) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:503: done (1 left) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:504: done (0 left) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:505: copy_from oid 26 from oid 6 current snap is 48 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:506: read oid 35 snap 42 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:506: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:39:06.179 INFO:tasks.rados.rados.0.smithi110.stdout:507: snap_create 2024-09-14T03:39:06.180 INFO:tasks.rados.rados.0.smithi110.stdout:506: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:39:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:505: finishing copy_from racing read to smithi11075562-26 2024-09-14T03:39:06.197 INFO:tasks.rados.rados.0.smithi110.stdout:505: finishing copy_from to smithi11075562-26 2024-09-14T03:39:06.197 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 119 (ObjNum 126 snap 33 seq_num 126) dirty exists 2024-09-14T03:39:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: pgmap v12: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 51 op/s 2024-09-14T03:39:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: pgmap v12: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 51 op/s 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:39:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:06 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: pgmap v12: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 51 op/s 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:39:06.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:06.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:06 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:07.106 INFO:tasks.rados.rados.0.smithi110.stdout:505: done (2 left) 2024-09-14T03:39:07.106 INFO:tasks.rados.rados.0.smithi110.stdout:506: done (1 left) 2024-09-14T03:39:07.106 INFO:tasks.rados.rados.0.smithi110.stdout:507: done (0 left) 2024-09-14T03:39:07.107 INFO:tasks.rados.rados.0.smithi110.stdout:508: setattr oid 25 current snap is 49 2024-09-14T03:39:07.107 INFO:tasks.rados.rados.0.smithi110.stdout:509: snap_create 2024-09-14T03:39:07.112 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 270 (ObjNum 129 snap 34 seq_num 129) dirty exists 2024-09-14T03:39:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:07 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi110 to 2480M 2024-09-14T03:39:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:07 smithi110 ceph-mon[35374]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:39:07.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:07 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi110 to 2480M 2024-09-14T03:39:07.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:07 smithi110 ceph-mon[27365]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:39:07.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:07 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi110 to 2480M 2024-09-14T03:39:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:07 smithi195 ceph-mon[30695]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:39:08.111 INFO:tasks.rados.rados.0.smithi110.stdout:508: done (1 left) 2024-09-14T03:39:08.112 INFO:tasks.rados.rados.0.smithi110.stdout:509: done (0 left) 2024-09-14T03:39:08.112 INFO:tasks.rados.rados.0.smithi110.stdout:510: rmattr oid 4 current snap is 50 2024-09-14T03:39:08.112 INFO:tasks.rados.rados.0.smithi110.stdout:510: done (0 left) 2024-09-14T03:39:08.112 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 50 2024-09-14T03:39:08.112 INFO:tasks.rados.rados.0.smithi110.stdout:511: seq_num 173 ranges {1417216=499712} 2024-09-14T03:39:08.122 INFO:tasks.rados.rados.0.smithi110.stdout:511: writing smithi11075562-39 from 1417216 to 1916928 tid 1 2024-09-14T03:39:08.122 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 50 2024-09-14T03:39:08.122 INFO:tasks.rados.rados.0.smithi110.stdout:512: seq_num 174 ranges {0=516096} 2024-09-14T03:39:08.125 INFO:tasks.rados.rados.0.smithi110.stdout:512: writing smithi11075562-37 from 0 to 516096 tid 1 2024-09-14T03:39:08.125 INFO:tasks.rados.rados.0.smithi110.stdout:513: read oid 33 snap 46 2024-09-14T03:39:08.125 INFO:tasks.rados.rados.0.smithi110.stdout:513: expect (ObjNum 108 snap 22 seq_num 108) 2024-09-14T03:39:08.125 INFO:tasks.rados.rados.0.smithi110.stdout:514: delete oid 23 current snap is 50 2024-09-14T03:39:08.125 INFO:tasks.rados.rados.0.smithi110.stdout:511: finishing write tid 1 to smithi11075562-39 2024-09-14T03:39:08.126 INFO:tasks.rados.rados.0.smithi110.stdout:511: finishing write tid 2 to smithi11075562-39 2024-09-14T03:39:08.126 INFO:tasks.rados.rados.0.smithi110.stdout:511: finishing write tid 3 to smithi11075562-39 2024-09-14T03:39:08.126 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 216 (ObjNum 173 snap 50 seq_num 173) dirty exists 2024-09-14T03:39:08.127 INFO:tasks.rados.rados.0.smithi110.stdout:511: left oid 39 (ObjNum 173 snap 50 seq_num 173) 2024-09-14T03:39:08.130 INFO:tasks.rados.rados.0.smithi110.stdout:511: done (3 left) 2024-09-14T03:39:08.131 INFO:tasks.rados.rados.0.smithi110.stdout:514: done (2 left) 2024-09-14T03:39:08.131 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 50 2024-09-14T03:39:08.131 INFO:tasks.rados.rados.0.smithi110.stdout:515: seq_num 176 ranges {3538944=557056} 2024-09-14T03:39:08.153 INFO:tasks.rados.rados.0.smithi110.stdout:515: writing smithi11075562-6 from 3538944 to 4096000 tid 1 2024-09-14T03:39:08.154 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 50 2024-09-14T03:39:08.154 INFO:tasks.rados.rados.0.smithi110.stdout:512: finishing write tid 1 to smithi11075562-37 2024-09-14T03:39:08.154 INFO:tasks.rados.rados.0.smithi110.stdout:516: seq_num 177 ranges {57344=573440} 2024-09-14T03:39:08.157 INFO:tasks.rados.rados.0.smithi110.stdout:516: writing smithi11075562-14 from 57344 to 630784 tid 1 2024-09-14T03:39:08.158 INFO:tasks.rados.rados.0.smithi110.stdout:517: rollback oid 17 current snap is 50 2024-09-14T03:39:08.158 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 45 2024-09-14T03:39:08.158 INFO:tasks.rados.rados.0.smithi110.stdout:512: finishing write tid 2 to smithi11075562-37 2024-09-14T03:39:08.158 INFO:tasks.rados.rados.0.smithi110.stdout:512: finishing write tid 3 to smithi11075562-37 2024-09-14T03:39:08.158 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 272 (ObjNum 174 snap 50 seq_num 174) dirty exists 2024-09-14T03:39:08.158 INFO:tasks.rados.rados.0.smithi110.stdout:512: left oid 37 (ObjNum 174 snap 50 seq_num 174) 2024-09-14T03:39:08.159 INFO:tasks.rados.rados.0.smithi110.stdout:513: expect (ObjNum 108 snap 22 seq_num 108) 2024-09-14T03:39:08.219 INFO:tasks.rados.rados.0.smithi110.stdout:515: finishing write tid 1 to smithi11075562-6 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:515: finishing write tid 2 to smithi11075562-6 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:515: finishing write tid 3 to smithi11075562-6 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 192 (ObjNum 176 snap 50 seq_num 176) dirty exists 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:515: left oid 6 (ObjNum 176 snap 50 seq_num 176) 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:517: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:517: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:512: done (4 left) 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:513: done (3 left) 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:515: done (2 left) 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:518: read oid 48 snap -1 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:518: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:519: copy_from oid 11 from oid 7 current snap is 50 2024-09-14T03:39:08.220 INFO:tasks.rados.rados.0.smithi110.stdout:517: finishing rollback tid 2 to smithi11075562-17 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 99 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:516: finishing write tid 1 to smithi11075562-14 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:516: finishing write tid 2 to smithi11075562-14 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:516: finishing write tid 3 to smithi11075562-14 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 219 (ObjNum 177 snap 50 seq_num 177) dirty exists 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:516: left oid 14 (ObjNum 177 snap 50 seq_num 177) 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:516: done (3 left) 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:517: done (2 left) 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 50 2024-09-14T03:39:08.221 INFO:tasks.rados.rados.0.smithi110.stdout:520: seq_num 178 ranges {3538944=589824} 2024-09-14T03:39:08.241 INFO:tasks.rados.rados.0.smithi110.stdout:520: writing smithi11075562-26 from 3538944 to 4128768 tid 1 2024-09-14T03:39:08.241 INFO:tasks.rados.rados.0.smithi110.stdout:521: read oid 38 snap -1 2024-09-14T03:39:08.242 INFO:tasks.rados.rados.0.smithi110.stdout:521: expect (ObjNum 142 snap 39 seq_num 142) 2024-09-14T03:39:08.242 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 50 2024-09-14T03:39:08.242 INFO:tasks.rados.rados.0.smithi110.stdout:522: seq_num 179 ranges {581632=606208} 2024-09-14T03:39:08.248 INFO:tasks.rados.rados.0.smithi110.stdout:522: writing smithi11075562-3 from 581632 to 1187840 tid 1 2024-09-14T03:39:08.248 INFO:tasks.rados.rados.0.smithi110.stdout:523: snap_create 2024-09-14T03:39:08.248 INFO:tasks.rados.rados.0.smithi110.stdout:518: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:39:08.253 INFO:tasks.rados.rados.0.smithi110.stdout:519: finishing copy_from racing read to smithi11075562-11 2024-09-14T03:39:08.253 INFO:tasks.rados.rados.0.smithi110.stdout:519: finishing copy_from to smithi11075562-11 2024-09-14T03:39:08.253 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 71 (ObjNum 154 snap 45 seq_num 154) dirty exists 2024-09-14T03:39:08.253 INFO:tasks.rados.rados.0.smithi110.stdout:520: finishing write tid 1 to smithi11075562-26 2024-09-14T03:39:08.253 INFO:tasks.rados.rados.0.smithi110.stdout:520: finishing write tid 2 to smithi11075562-26 2024-09-14T03:39:08.253 INFO:tasks.rados.rados.0.smithi110.stdout:520: finishing write tid 3 to smithi11075562-26 2024-09-14T03:39:08.254 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 121 (ObjNum 178 snap 50 seq_num 178) dirty exists 2024-09-14T03:39:08.254 INFO:tasks.rados.rados.0.smithi110.stdout:520: left oid 26 (ObjNum 178 snap 50 seq_num 178) 2024-09-14T03:39:08.254 INFO:tasks.rados.rados.0.smithi110.stdout:522: finishing write tid 1 to smithi11075562-3 2024-09-14T03:39:08.254 INFO:tasks.rados.rados.0.smithi110.stdout:522: finishing write tid 2 to smithi11075562-3 2024-09-14T03:39:08.254 INFO:tasks.rados.rados.0.smithi110.stdout:522: finishing write tid 3 to smithi11075562-3 2024-09-14T03:39:08.254 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 132 (ObjNum 179 snap 50 seq_num 179) dirty exists 2024-09-14T03:39:08.254 INFO:tasks.rados.rados.0.smithi110.stdout:522: left oid 3 (ObjNum 179 snap 50 seq_num 179) 2024-09-14T03:39:08.260 INFO:tasks.rados.rados.0.smithi110.stdout:521: expect (ObjNum 142 snap 39 seq_num 142) 2024-09-14T03:39:08.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:08 smithi195 ceph-mon[30695]: pgmap v15: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 51 op/s 2024-09-14T03:39:08.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:08 smithi195 ceph-mon[30695]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:39:08.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:08 smithi110 ceph-mon[35374]: pgmap v15: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 51 op/s 2024-09-14T03:39:08.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:08 smithi110 ceph-mon[35374]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:39:08.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:08 smithi110 ceph-mon[27365]: pgmap v15: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 51 op/s 2024-09-14T03:39:08.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:08 smithi110 ceph-mon[27365]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:39:09.113 INFO:tasks.rados.rados.0.smithi110.stdout:518: done (5 left) 2024-09-14T03:39:09.113 INFO:tasks.rados.rados.0.smithi110.stdout:519: done (4 left) 2024-09-14T03:39:09.113 INFO:tasks.rados.rados.0.smithi110.stdout:520: done (3 left) 2024-09-14T03:39:09.113 INFO:tasks.rados.rados.0.smithi110.stdout:521: done (2 left) 2024-09-14T03:39:09.114 INFO:tasks.rados.rados.0.smithi110.stdout:522: done (1 left) 2024-09-14T03:39:09.114 INFO:tasks.rados.rados.0.smithi110.stdout:523: done (0 left) 2024-09-14T03:39:09.114 INFO:tasks.rados.rados.0.smithi110.stdout:524: read oid 37 snap -1 2024-09-14T03:39:09.114 INFO:tasks.rados.rados.0.smithi110.stdout:524: expect (ObjNum 174 snap 50 seq_num 174) 2024-09-14T03:39:09.114 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 51 2024-09-14T03:39:09.114 INFO:tasks.rados.rados.0.smithi110.stdout:525: seq_num 180 ranges {794624=622592} 2024-09-14T03:39:09.121 INFO:tasks.rados.rados.0.smithi110.stdout:525: writing smithi11075562-42 from 794624 to 1417216 tid 1 2024-09-14T03:39:09.122 INFO:tasks.rados.rados.0.smithi110.stdout:526: read oid 4 snap 24 2024-09-14T03:39:09.122 INFO:tasks.rados.rados.0.smithi110.stdout:526: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:39:09.122 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 51 2024-09-14T03:39:09.122 INFO:tasks.rados.rados.0.smithi110.stdout:527: seq_num 181 ranges {3915776=638976} 2024-09-14T03:39:09.146 INFO:tasks.rados.rados.0.smithi110.stdout:527: writing smithi11075562-13 from 3915776 to 4554752 tid 1 2024-09-14T03:39:09.146 INFO:tasks.rados.rados.0.smithi110.stdout:528: read oid 38 snap -1 2024-09-14T03:39:09.146 INFO:tasks.rados.rados.0.smithi110.stdout:528: expect (ObjNum 142 snap 39 seq_num 142) 2024-09-14T03:39:09.146 INFO:tasks.rados.rados.0.smithi110.stdout:529: rmattr oid 22 current snap is 51 2024-09-14T03:39:09.146 INFO:tasks.rados.rados.0.smithi110.stdout:524: expect (ObjNum 174 snap 50 seq_num 174) 2024-09-14T03:39:09.163 INFO:tasks.rados.rados.0.smithi110.stdout:526: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:39:09.164 INFO:tasks.rados.rados.0.smithi110.stdout:525: finishing write tid 1 to smithi11075562-42 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:525: finishing write tid 2 to smithi11075562-42 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:525: finishing write tid 3 to smithi11075562-42 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 187 (ObjNum 180 snap 51 seq_num 180) dirty exists 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:525: left oid 42 (ObjNum 180 snap 51 seq_num 180) 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 163 (ObjNum 163 snap 48 seq_num 163) dirty exists 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:527: finishing write tid 1 to smithi11075562-13 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:527: finishing write tid 2 to smithi11075562-13 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:527: finishing write tid 3 to smithi11075562-13 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 190 (ObjNum 181 snap 51 seq_num 181) dirty exists 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:527: left oid 13 (ObjNum 181 snap 51 seq_num 181) 2024-09-14T03:39:09.165 INFO:tasks.rados.rados.0.smithi110.stdout:528: expect (ObjNum 142 snap 39 seq_num 142) 2024-09-14T03:39:09.483 INFO:tasks.rados.rados.0.smithi110.stdout:524: done (5 left) 2024-09-14T03:39:09.483 INFO:tasks.rados.rados.0.smithi110.stdout:525: done (4 left) 2024-09-14T03:39:09.483 INFO:tasks.rados.rados.0.smithi110.stdout:526: done (3 left) 2024-09-14T03:39:09.483 INFO:tasks.rados.rados.0.smithi110.stdout:527: done (2 left) 2024-09-14T03:39:09.483 INFO:tasks.rados.rados.0.smithi110.stdout:528: done (1 left) 2024-09-14T03:39:09.483 INFO:tasks.rados.rados.0.smithi110.stdout:529: done (0 left) 2024-09-14T03:39:09.483 INFO:tasks.rados.rados.0.smithi110.stdout:530: delete oid 19 current snap is 51 2024-09-14T03:39:09.485 INFO:tasks.rados.rados.0.smithi110.stdout:530: done (0 left) 2024-09-14T03:39:09.485 INFO:tasks.rados.rados.0.smithi110.stdout:531: delete oid 27 current snap is 51 2024-09-14T03:39:09.486 INFO:tasks.rados.rados.0.smithi110.stdout:531: done (0 left) 2024-09-14T03:39:09.486 INFO:tasks.rados.rados.0.smithi110.stdout:532: rmattr oid 6 current snap is 51 2024-09-14T03:39:09.486 INFO:tasks.rados.rados.0.smithi110.stdout:532: done (0 left) 2024-09-14T03:39:09.486 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 51 2024-09-14T03:39:09.486 INFO:tasks.rados.rados.0.smithi110.stdout:533: seq_num 184 ranges {630784=688128} 2024-09-14T03:39:09.494 INFO:tasks.rados.rados.0.smithi110.stdout:533: writing smithi11075562-14 from 630784 to 1318912 tid 1 2024-09-14T03:39:09.494 INFO:tasks.rados.rados.0.smithi110.stdout:534: read oid 4 snap -1 2024-09-14T03:39:09.494 INFO:tasks.rados.rados.0.smithi110.stdout:534: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:39:09.494 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 51 2024-09-14T03:39:09.494 INFO:tasks.rados.rados.0.smithi110.stdout:535: seq_num 185 ranges {827392=704512} 2024-09-14T03:39:09.502 INFO:tasks.rados.rados.0.smithi110.stdout:535: writing smithi11075562-50 from 827392 to 1531904 tid 1 2024-09-14T03:39:09.502 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 51 2024-09-14T03:39:09.502 INFO:tasks.rados.rados.0.smithi110.stdout:536: seq_num 186 ranges {0=720896} 2024-09-14T03:39:09.507 INFO:tasks.rados.rados.0.smithi110.stdout:536: writing smithi11075562-35 from 0 to 720896 tid 1 2024-09-14T03:39:09.507 INFO:tasks.rados.rados.0.smithi110.stdout:537: snap_remove snap 30 2024-09-14T03:39:09.507 INFO:tasks.rados.rados.0.smithi110.stdout:534: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:39:09.508 INFO:tasks.rados.rados.0.smithi110.stdout:533: finishing write tid 1 to smithi11075562-14 2024-09-14T03:39:09.508 INFO:tasks.rados.rados.0.smithi110.stdout:533: finishing write tid 2 to smithi11075562-14 2024-09-14T03:39:09.509 INFO:tasks.rados.rados.0.smithi110.stdout:533: finishing write tid 3 to smithi11075562-14 2024-09-14T03:39:09.509 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 222 (ObjNum 184 snap 51 seq_num 184) dirty exists 2024-09-14T03:39:09.509 INFO:tasks.rados.rados.0.smithi110.stdout:533: left oid 14 (ObjNum 184 snap 51 seq_num 184) 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:533: done (4 left) 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:534: done (3 left) 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:537: done (2 left) 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:538: snap_create 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:535: finishing write tid 1 to smithi11075562-50 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:535: finishing write tid 2 to smithi11075562-50 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:535: finishing write tid 3 to smithi11075562-50 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 20 (ObjNum 185 snap 51 seq_num 185) dirty exists 2024-09-14T03:39:10.030 INFO:tasks.rados.rados.0.smithi110.stdout:535: left oid 50 (ObjNum 185 snap 51 seq_num 185) 2024-09-14T03:39:10.031 INFO:tasks.rados.rados.0.smithi110.stdout:536: finishing write tid 1 to smithi11075562-35 2024-09-14T03:39:10.031 INFO:tasks.rados.rados.0.smithi110.stdout:536: finishing write tid 2 to smithi11075562-35 2024-09-14T03:39:10.031 INFO:tasks.rados.rados.0.smithi110.stdout:536: finishing write tid 3 to smithi11075562-35 2024-09-14T03:39:10.031 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 192 (ObjNum 186 snap 51 seq_num 186) dirty exists 2024-09-14T03:39:10.031 INFO:tasks.rados.rados.0.smithi110.stdout:536: left oid 35 (ObjNum 186 snap 51 seq_num 186) 2024-09-14T03:39:10.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:39:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:39:10.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:39:10.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: pgmap v18: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:10.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: Adjusting osd_memory_target on smithi195 to 2480M 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[27365]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: pgmap v18: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: Adjusting osd_memory_target on smithi195 to 2480M 2024-09-14T03:39:10.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:10 smithi110 ceph-mon[35374]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:39:10] "GET /metrics HTTP/1.1" 200 35872 "" "Prometheus/2.43.0" 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: pgmap v18: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: Adjusting osd_memory_target on smithi195 to 2480M 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:10.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:10 smithi195 ceph-mon[30695]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:39:11.030 INFO:tasks.rados.rados.0.smithi110.stdout:535: done (2 left) 2024-09-14T03:39:11.030 INFO:tasks.rados.rados.0.smithi110.stdout:536: done (1 left) 2024-09-14T03:39:11.030 INFO:tasks.rados.rados.0.smithi110.stdout:538: done (0 left) 2024-09-14T03:39:11.031 INFO:tasks.rados.rados.0.smithi110.stdout:539: snap_remove snap 24 2024-09-14T03:39:11.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:11.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:11.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:11.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: Deploying daemon alertmanager.a on smithi110 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: Deploying daemon alertmanager.a on smithi110 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:39:11.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[27365]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:39:11.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:11 smithi110 ceph-mon[35374]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:39:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:39:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:39:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: Deploying daemon alertmanager.a on smithi110 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:39:11.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:11.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:11.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:11.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:11 smithi195 ceph-mon[30695]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:39:11.650 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:39:12.035 INFO:tasks.rados.rados.0.smithi110.stdout:539: done (0 left) 2024-09-14T03:39:12.035 INFO:tasks.rados.rados.0.smithi110.stdout:540: read oid 16 snap 46 2024-09-14T03:39:12.035 INFO:tasks.rados.rados.0.smithi110.stdout:540: expect (ObjNum 69 snap 8 seq_num 69) 2024-09-14T03:39:12.035 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 52 2024-09-14T03:39:12.035 INFO:tasks.rados.rados.0.smithi110.stdout:541: seq_num 187 ranges {761856=737280} 2024-09-14T03:39:12.044 INFO:tasks.rados.rados.0.smithi110.stdout:541: writing smithi11075562-2 from 761856 to 1499136 tid 1 2024-09-14T03:39:12.044 INFO:tasks.rados.rados.0.smithi110.stdout:542: setattr oid 49 current snap is 52 2024-09-14T03:39:12.047 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 52 2024-09-14T03:39:12.047 INFO:tasks.rados.rados.0.smithi110.stdout:543: seq_num 188 ranges {0=753664} 2024-09-14T03:39:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:543: writing smithi11075562-45 from 0 to 753664 tid 1 2024-09-14T03:39:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:544: read oid 35 snap -1 2024-09-14T03:39:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:544: expect (ObjNum 186 snap 51 seq_num 186) 2024-09-14T03:39:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:545: read oid 14 snap -1 2024-09-14T03:39:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:545: expect (ObjNum 184 snap 51 seq_num 184) 2024-09-14T03:39:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:546: copy_from oid 19 from oid 32 current snap is 52 2024-09-14T03:39:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:541: finishing write tid 1 to smithi11075562-2 2024-09-14T03:39:12.054 INFO:tasks.rados.rados.0.smithi110.stdout:541: finishing write tid 2 to smithi11075562-2 2024-09-14T03:39:12.054 INFO:tasks.rados.rados.0.smithi110.stdout:541: finishing write tid 3 to smithi11075562-2 2024-09-14T03:39:12.054 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 147 (ObjNum 187 snap 52 seq_num 187) dirty exists 2024-09-14T03:39:12.054 INFO:tasks.rados.rados.0.smithi110.stdout:541: left oid 2 (ObjNum 187 snap 52 seq_num 187) 2024-09-14T03:39:12.054 INFO:tasks.rados.rados.0.smithi110.stdout:541: done (6 left) 2024-09-14T03:39:12.054 INFO:tasks.rados.rados.0.smithi110.stdout:547: rollback oid 3 current snap is 52 2024-09-14T03:39:12.054 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 3 to 39 2024-09-14T03:39:12.054 INFO:tasks.rados.rados.0.smithi110.stdout:548: snap_remove snap 50 2024-09-14T03:39:12.292 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:39:12.292 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (3m) 7s ago 4m 29.1M - 0.25.0 c8568f914cd2 764997e49f35 2024-09-14T03:39:12.292 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (3m) 6s ago 3m 89.0M - 9.4.7 954c08fa6188 67ea5066f839 2024-09-14T03:39:12.292 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (5m) 7s ago 5m 17.3M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:39:12.292 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (5m) 6s ago 5m 15.8M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:39:12.292 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283 running (33s) 6s ago 8m 499M - 19.3.0-4907-ga2782315 8801d2529e43 2372ccbce64c 2024-09-14T03:39:12.292 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:9283,8765 running (10m) 7s ago 10m 309M - 17.2.7-1629-geb16b97b ab51d1bebd12 a83ea41d9e62 2024-09-14T03:39:12.292 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (10m) 7s ago 10m 54.1M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 c1073676c66c 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (9m) 6s ago 9m 44.5M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 b6415a0de807 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (8m) 7s ago 8m 42.6M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 8e2894672cd8 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (3m) 7s ago 3m 15.6M - 1.5.0 0da6a335fe13 986afca49027 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (3m) 6s ago 3m 16.4M - 1.5.0 0da6a335fe13 67a66167cf52 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (8m) 7s ago 8m 78.4M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (8m) 7s ago 8m 80.5M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (7m) 7s ago 7m 79.6M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (7m) 7s ago 7m 72.1M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (6m) 6s ago 6m 110M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (6m) 6s ago 6m 85.7M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (6m) 6s ago 6m 68.5M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (5m) 6s ago 5m 65.8M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:39:12.293 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (3m) 6s ago 4m 54.1M - 2.43.0 a07b618ecd1d 4460d604b8f6 2024-09-14T03:39:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:12 smithi110 ceph-mon[35374]: mgrmap e29: x(active, since 11s), standbys: y 2024-09-14T03:39:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:12 smithi110 ceph-mon[35374]: pgmap v21: 121 pgs: 121 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T03:39:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:12 smithi110 ceph-mon[35374]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:39:12.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:12 smithi110 ceph-mon[27365]: mgrmap e29: x(active, since 11s), standbys: y 2024-09-14T03:39:12.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:12 smithi110 ceph-mon[27365]: pgmap v21: 121 pgs: 121 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T03:39:12.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:12 smithi110 ceph-mon[27365]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:39:12.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:12 smithi195 ceph-mon[30695]: mgrmap e29: x(active, since 11s), standbys: y 2024-09-14T03:39:12.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:12 smithi195 ceph-mon[30695]: pgmap v21: 121 pgs: 121 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T03:39:12.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:12 smithi195 ceph-mon[30695]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:39:12.645 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:39:12.645 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:39:12.645 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:39:12.646 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 14, 2024-09-14T03:39:12.647 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:39:12.647 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:39:12.647 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:39:12.974 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:39:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:39:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:39:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:39:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [], 2024-09-14T03:39:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "1/20 daemons upgraded", 2024-09-14T03:39:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "", 2024-09-14T03:39:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:39:12.975 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:39:13.037 INFO:tasks.rados.rados.0.smithi110.stdout:548: done (7 left) 2024-09-14T03:39:13.037 INFO:tasks.rados.rados.0.smithi110.stdout:549: read oid 18 snap -1 2024-09-14T03:39:13.037 INFO:tasks.rados.rados.0.smithi110.stdout:549: expect deleted 2024-09-14T03:39:13.037 INFO:tasks.rados.rados.0.smithi110.stdout:550: rollback oid 7 current snap is 52 2024-09-14T03:39:13.037 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 7 to 49 2024-09-14T03:39:13.037 INFO:tasks.rados.rados.0.smithi110.stdout:551: delete oid 48 current snap is 52 2024-09-14T03:39:13.037 INFO:tasks.rados.rados.0.smithi110.stdout:540: expect (ObjNum 69 snap 8 seq_num 69) 2024-09-14T03:39:13.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 214 (ObjNum 160 snap 47 seq_num 160) dirty exists 2024-09-14T03:39:13.078 INFO:tasks.rados.rados.0.smithi110.stdout:547: finishing rollback tid 0 to smithi11075562-3 2024-09-14T03:39:13.078 INFO:tasks.rados.rados.0.smithi110.stdout:547: finishing rollback tid 1 to smithi11075562-3 2024-09-14T03:39:13.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 150 (ObjNum 49 snap 0 seq_num 64916144) dirty dne 2024-09-14T03:39:13.078 INFO:tasks.rados.rados.0.smithi110.stdout:543: finishing write tid 1 to smithi11075562-45 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:540: done (9 left) 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:542: done (8 left) 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:547: done (7 left) 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:551: done (6 left) 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:552: rmattr oid 13 current snap is 52 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:543: finishing write tid 2 to smithi11075562-45 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:543: finishing write tid 3 to smithi11075562-45 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 292 (ObjNum 188 snap 52 seq_num 188) dirty exists 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:543: left oid 45 (ObjNum 188 snap 52 seq_num 188) 2024-09-14T03:39:13.079 INFO:tasks.rados.rados.0.smithi110.stdout:544: expect (ObjNum 186 snap 51 seq_num 186) 2024-09-14T03:39:13.097 INFO:tasks.rados.rados.0.smithi110.stdout:545: expect (ObjNum 184 snap 51 seq_num 184) 2024-09-14T03:39:13.172 INFO:tasks.rados.rados.0.smithi110.stdout:546: finishing copy_from racing read to smithi11075562-19 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:546: finishing copy_from to smithi11075562-19 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 53 (ObjNum 109 snap 25 seq_num 109) dirty exists 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:543: done (6 left) 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:544: done (5 left) 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:545: done (4 left) 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:546: done (3 left) 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:553: snap_create 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:550: finishing rollback tid 0 to smithi11075562-7 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:550: finishing rollback tid 1 to smithi11075562-7 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:550: finishing rollback tid 2 to smithi11075562-7 2024-09-14T03:39:13.173 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 91 (ObjNum 154 snap 45 seq_num 154) dirty exists 2024-09-14T03:39:13.174 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 222 (ObjNum 181 snap 51 seq_num 181) dirty exists 2024-09-14T03:39:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[35374]: from='client.15066 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[35374]: from='client.15072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[35374]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/2813981416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:39:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[35374]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:39:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[27365]: from='client.15066 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[27365]: from='client.15072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[27365]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/2813981416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:39:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:13 smithi110 ceph-mon[27365]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:39:13.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:13 smithi195 ceph-mon[30695]: from='client.15066 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:13 smithi195 ceph-mon[30695]: from='client.15072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:13 smithi195 ceph-mon[30695]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:13 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/2813981416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:39:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:13 smithi195 ceph-mon[30695]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:39:14.137 INFO:tasks.rados.rados.0.smithi110.stdout:549: done (3 left) 2024-09-14T03:39:14.137 INFO:tasks.rados.rados.0.smithi110.stdout:550: done (2 left) 2024-09-14T03:39:14.137 INFO:tasks.rados.rados.0.smithi110.stdout:552: done (1 left) 2024-09-14T03:39:14.137 INFO:tasks.rados.rados.0.smithi110.stdout:553: done (0 left) 2024-09-14T03:39:14.137 INFO:tasks.rados.rados.0.smithi110.stdout:554: delete oid 37 current snap is 53 2024-09-14T03:39:14.141 INFO:tasks.rados.rados.0.smithi110.stdout:554: done (0 left) 2024-09-14T03:39:14.141 INFO:tasks.rados.rados.0.smithi110.stdout:555: read oid 28 snap 52 2024-09-14T03:39:14.141 INFO:tasks.rados.rados.0.smithi110.stdout:555: expect (ObjNum 168 snap 48 seq_num 168) 2024-09-14T03:39:14.141 INFO:tasks.rados.rados.0.smithi110.stdout:556: read oid 2 snap 48 2024-09-14T03:39:14.141 INFO:tasks.rados.rados.0.smithi110.stdout:556: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:39:14.141 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 53 2024-09-14T03:39:14.141 INFO:tasks.rados.rados.0.smithi110.stdout:557: seq_num 191 ranges {0=802816} 2024-09-14T03:39:14.146 INFO:tasks.rados.rados.0.smithi110.stdout:557: writing smithi11075562-5 from 0 to 802816 tid 1 2024-09-14T03:39:14.146 INFO:tasks.rados.rados.0.smithi110.stdout:558: rmattr oid 48 current snap is 53 2024-09-14T03:39:14.146 INFO:tasks.rados.rados.0.smithi110.stdout:558: done (3 left) 2024-09-14T03:39:14.146 INFO:tasks.rados.rados.0.smithi110.stdout:559: read oid 18 snap -1 2024-09-14T03:39:14.147 INFO:tasks.rados.rados.0.smithi110.stdout:559: expect deleted 2024-09-14T03:39:14.147 INFO:tasks.rados.rados.0.smithi110.stdout:560: snap_create 2024-09-14T03:39:14.147 INFO:tasks.rados.rados.0.smithi110.stdout:555: expect (ObjNum 168 snap 48 seq_num 168) 2024-09-14T03:39:14.160 INFO:tasks.rados.rados.0.smithi110.stdout:556: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:39:14.181 INFO:tasks.rados.rados.0.smithi110.stdout:557: finishing write tid 1 to smithi11075562-5 2024-09-14T03:39:14.181 INFO:tasks.rados.rados.0.smithi110.stdout:557: finishing write tid 2 to smithi11075562-5 2024-09-14T03:39:14.181 INFO:tasks.rados.rados.0.smithi110.stdout:557: finishing write tid 3 to smithi11075562-5 2024-09-14T03:39:14.181 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 296 (ObjNum 191 snap 53 seq_num 191) dirty exists 2024-09-14T03:39:14.181 INFO:tasks.rados.rados.0.smithi110.stdout:557: left oid 5 (ObjNum 191 snap 53 seq_num 191) 2024-09-14T03:39:14.445 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:14 smithi110 ceph-mon[35374]: from='client.15090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:14 smithi110 ceph-mon[35374]: pgmap v24: 121 pgs: 121 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T03:39:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:14 smithi110 ceph-mon[27365]: from='client.15090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:14 smithi110 ceph-mon[27365]: pgmap v24: 121 pgs: 121 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T03:39:14.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:14 smithi195 ceph-mon[30695]: from='client.15090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:14.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:14 smithi195 ceph-mon[30695]: pgmap v24: 121 pgs: 121 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T03:39:15.030 INFO:tasks.rados.rados.0.smithi110.stdout:555: done (4 left) 2024-09-14T03:39:15.030 INFO:tasks.rados.rados.0.smithi110.stdout:556: done (3 left) 2024-09-14T03:39:15.030 INFO:tasks.rados.rados.0.smithi110.stdout:557: done (2 left) 2024-09-14T03:39:15.031 INFO:tasks.rados.rados.0.smithi110.stdout:559: done (1 left) 2024-09-14T03:39:15.031 INFO:tasks.rados.rados.0.smithi110.stdout:560: done (0 left) 2024-09-14T03:39:15.031 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 54 2024-09-14T03:39:15.031 INFO:tasks.rados.rados.0.smithi110.stdout:561: seq_num 192 ranges {0=417792,417792=401408} 2024-09-14T03:39:15.032 INFO:tasks.rados.rados.0.smithi110.stdout:561: writing smithi11075562-3 from 0 to 417792 tid 1 2024-09-14T03:39:15.035 INFO:tasks.rados.rados.0.smithi110.stdout:561: writing smithi11075562-3 from 417792 to 819200 tid 2 2024-09-14T03:39:15.035 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 54 2024-09-14T03:39:15.035 INFO:tasks.rados.rados.0.smithi110.stdout:562: seq_num 193 ranges {4382720=434176,4816896=401408} 2024-09-14T03:39:15.062 INFO:tasks.rados.rados.0.smithi110.stdout:562: writing smithi11075562-49 from 4382720 to 4816896 tid 1 2024-09-14T03:39:15.064 INFO:tasks.rados.rados.0.smithi110.stdout:562: writing smithi11075562-49 from 4816896 to 5218304 tid 2 2024-09-14T03:39:15.064 INFO:tasks.rados.rados.0.smithi110.stdout:563: read oid 32 snap 53 2024-09-14T03:39:15.064 INFO:tasks.rados.rados.0.smithi110.stdout:563: expect (ObjNum 109 snap 25 seq_num 109) 2024-09-14T03:39:15.064 INFO:tasks.rados.rados.0.smithi110.stdout:564: rollback oid 50 current snap is 54 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 39 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:561: finishing write tid 1 to smithi11075562-3 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:561: finishing write tid 2 to smithi11075562-3 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:561: finishing write tid 3 to smithi11075562-3 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:561: finishing write tid 4 to smithi11075562-3 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 157 (ObjNum 192 snap 54 seq_num 192) dirty exists 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:561: left oid 3 (ObjNum 192 snap 54 seq_num 192) 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:561: done (3 left) 2024-09-14T03:39:15.065 INFO:tasks.rados.rados.0.smithi110.stdout:565: snap_create 2024-09-14T03:39:15.067 INFO:tasks.rados.rados.0.smithi110.stdout:564: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:39:15.067 INFO:tasks.rados.rados.0.smithi110.stdout:562: finishing write tid 1 to smithi11075562-49 2024-09-14T03:39:15.068 INFO:tasks.rados.rados.0.smithi110.stdout:562: finishing write tid 2 to smithi11075562-49 2024-09-14T03:39:15.068 INFO:tasks.rados.rados.0.smithi110.stdout:564: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:39:15.068 INFO:tasks.rados.rados.0.smithi110.stdout:564: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:39:15.068 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 30 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:39:15.069 INFO:tasks.rados.rados.0.smithi110.stdout:562: finishing write tid 3 to smithi11075562-49 2024-09-14T03:39:15.069 INFO:tasks.rados.rados.0.smithi110.stdout:562: finishing write tid 4 to smithi11075562-49 2024-09-14T03:39:15.070 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 226 (ObjNum 193 snap 54 seq_num 193) dirty exists 2024-09-14T03:39:15.070 INFO:tasks.rados.rados.0.smithi110.stdout:562: left oid 49 (ObjNum 193 snap 54 seq_num 193) 2024-09-14T03:39:15.072 INFO:tasks.rados.rados.0.smithi110.stdout:563: expect (ObjNum 109 snap 25 seq_num 109) 2024-09-14T03:39:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:15 smithi110 ceph-mon[35374]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:39:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:15 smithi110 ceph-mon[35374]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:39:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:15 smithi110 ceph-mon[27365]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:39:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:15 smithi110 ceph-mon[27365]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:39:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:15 smithi195 ceph-mon[30695]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:39:15.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:15 smithi195 ceph-mon[30695]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:39:16.031 INFO:tasks.rados.rados.0.smithi110.stdout:562: done (3 left) 2024-09-14T03:39:16.031 INFO:tasks.rados.rados.0.smithi110.stdout:563: done (2 left) 2024-09-14T03:39:16.031 INFO:tasks.rados.rados.0.smithi110.stdout:564: done (1 left) 2024-09-14T03:39:16.032 INFO:tasks.rados.rados.0.smithi110.stdout:565: done (0 left) 2024-09-14T03:39:16.032 INFO:tasks.rados.rados.0.smithi110.stdout:566: read oid 22 snap -1 2024-09-14T03:39:16.032 INFO:tasks.rados.rados.0.smithi110.stdout:566: expect (ObjNum 163 snap 48 seq_num 163) 2024-09-14T03:39:16.032 INFO:tasks.rados.rados.0.smithi110.stdout:567: rollback oid 50 current snap is 55 2024-09-14T03:39:16.032 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 20 2024-09-14T03:39:16.032 INFO:tasks.rados.rados.0.smithi110.stdout:568: delete oid 20 current snap is 55 2024-09-14T03:39:16.036 INFO:tasks.rados.rados.0.smithi110.stdout:567: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:39:16.036 INFO:tasks.rados.rados.0.smithi110.stdout:567: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:568: done (2 left) 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:569: rollback oid 31 current snap is 55 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 39 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:567: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 34 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:567: done (2 left) 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:570: read oid 39 snap -1 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:570: expect (ObjNum 173 snap 50 seq_num 173) 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:571: read oid 36 snap -1 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:571: expect (ObjNum 133 snap 36 seq_num 133) 2024-09-14T03:39:16.037 INFO:tasks.rados.rados.0.smithi110.stdout:572: snap_create 2024-09-14T03:39:16.040 INFO:tasks.rados.rados.0.smithi110.stdout:569: finishing rollback tid 0 to smithi11075562-31 2024-09-14T03:39:16.041 INFO:tasks.rados.rados.0.smithi110.stdout:569: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:39:16.041 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 64 (ObjNum 65 snap 0 seq_num 1499878498) dirty dne 2024-09-14T03:39:16.046 INFO:tasks.rados.rados.0.smithi110.stdout:570: expect (ObjNum 173 snap 50 seq_num 173) 2024-09-14T03:39:16.127 INFO:tasks.rados.rados.0.smithi110.stdout:566: expect (ObjNum 163 snap 48 seq_num 163) 2024-09-14T03:39:16.329 INFO:tasks.rados.rados.0.smithi110.stdout:571: expect (ObjNum 133 snap 36 seq_num 133) 2024-09-14T03:39:16.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:16 smithi110 ceph-mon[35374]: pgmap v27: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 31 op/s 2024-09-14T03:39:16.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:16 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:16.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:16 smithi110 ceph-mon[35374]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:39:16.376 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:16 smithi110 ceph-mon[27365]: pgmap v27: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 31 op/s 2024-09-14T03:39:16.376 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:16 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:16.376 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:16 smithi110 ceph-mon[27365]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:39:16.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:16 smithi195 ceph-mon[30695]: pgmap v27: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 31 op/s 2024-09-14T03:39:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:16 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:16 smithi195 ceph-mon[30695]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:39:17.032 INFO:tasks.rados.rados.0.smithi110.stdout:566: done (4 left) 2024-09-14T03:39:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:569: done (3 left) 2024-09-14T03:39:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:570: done (2 left) 2024-09-14T03:39:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:571: done (1 left) 2024-09-14T03:39:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:572: done (0 left) 2024-09-14T03:39:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 56 2024-09-14T03:39:17.033 INFO:tasks.rados.rados.0.smithi110.stdout:573: seq_num 195 ranges {6864896=475136,7340032=401408} 2024-09-14T03:39:17.071 INFO:tasks.rados.rados.0.smithi110.stdout:573: writing smithi11075562-38 from 6864896 to 7340032 tid 1 2024-09-14T03:39:17.074 INFO:tasks.rados.rados.0.smithi110.stdout:573: writing smithi11075562-38 from 7340032 to 7741440 tid 2 2024-09-14T03:39:17.074 INFO:tasks.rados.rados.0.smithi110.stdout:574: setattr oid 26 current snap is 56 2024-09-14T03:39:17.076 INFO:tasks.rados.rados.0.smithi110.stdout:573: finishing write tid 1 to smithi11075562-38 2024-09-14T03:39:17.076 INFO:tasks.rados.rados.0.smithi110.stdout:573: finishing write tid 2 to smithi11075562-38 2024-09-14T03:39:17.077 INFO:tasks.rados.rados.0.smithi110.stdout:573: finishing write tid 3 to smithi11075562-38 2024-09-14T03:39:17.077 INFO:tasks.rados.rados.0.smithi110.stdout:575: delete oid 43 current snap is 56 2024-09-14T03:39:17.077 INFO:tasks.rados.rados.0.smithi110.stdout:573: finishing write tid 4 to smithi11075562-38 2024-09-14T03:39:17.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 95 (ObjNum 195 snap 56 seq_num 195) dirty exists 2024-09-14T03:39:17.078 INFO:tasks.rados.rados.0.smithi110.stdout:573: left oid 38 (ObjNum 195 snap 56 seq_num 195) 2024-09-14T03:39:17.079 INFO:tasks.rados.rados.0.smithi110.stdout:573: done (2 left) 2024-09-14T03:39:17.079 INFO:tasks.rados.rados.0.smithi110.stdout:575: done (1 left) 2024-09-14T03:39:17.079 INFO:tasks.rados.rados.0.smithi110.stdout:576: snap_remove snap 49 2024-09-14T03:39:17.760 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:17 smithi110 systemd[1]: Stopping Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:18.035 INFO:tasks.rados.rados.0.smithi110.stdout:576: done (1 left) 2024-09-14T03:39:18.035 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 56 2024-09-14T03:39:18.035 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 122 (ObjNum 178 snap 50 seq_num 178) dirty exists 2024-09-14T03:39:18.035 INFO:tasks.rados.rados.0.smithi110.stdout:577: seq_num 197 ranges {3293184=507904,3801088=401408} 2024-09-14T03:39:18.037 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:17 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[71405]: ts=2024-09-14T03:39:17.758Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:39:18.037 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:17 smithi110 podman[77630]: 2024-09-14 03:39:17.799620996 +0000 UTC m=+0.151511642 container died 764997e49f35d23edf747b6ce67f2dab482d406113349b9f5709722242ad63be (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:18.056 INFO:tasks.rados.rados.0.smithi110.stdout:577: writing smithi11075562-9 from 3293184 to 3801088 tid 1 2024-09-14T03:39:18.060 INFO:tasks.rados.rados.0.smithi110.stdout:577: writing smithi11075562-9 from 3801088 to 4202496 tid 2 2024-09-14T03:39:18.060 INFO:tasks.rados.rados.0.smithi110.stdout:574: done (1 left) 2024-09-14T03:39:18.061 INFO:tasks.rados.rados.0.smithi110.stdout:578: snap_create 2024-09-14T03:39:18.061 INFO:tasks.rados.rados.0.smithi110.stdout:577: finishing write tid 1 to smithi11075562-9 2024-09-14T03:39:18.061 INFO:tasks.rados.rados.0.smithi110.stdout:577: finishing write tid 2 to smithi11075562-9 2024-09-14T03:39:18.061 INFO:tasks.rados.rados.0.smithi110.stdout:577: finishing write tid 3 to smithi11075562-9 2024-09-14T03:39:18.062 INFO:tasks.rados.rados.0.smithi110.stdout:577: finishing write tid 4 to smithi11075562-9 2024-09-14T03:39:18.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 172 (ObjNum 197 snap 56 seq_num 197) dirty exists 2024-09-14T03:39:18.062 INFO:tasks.rados.rados.0.smithi110.stdout:577: left oid 9 (ObjNum 197 snap 56 seq_num 197) 2024-09-14T03:39:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:18 smithi110 ceph-mon[35374]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:39:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:18 smithi110 ceph-mon[35374]: pgmap v30: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 31 op/s 2024-09-14T03:39:18.446 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:18 smithi110 podman[77630]: 2024-09-14 03:39:18.234584896 +0000 UTC m=+0.586475542 container cleanup 764997e49f35d23edf747b6ce67f2dab482d406113349b9f5709722242ad63be (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:18.446 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:18 smithi110 bash[77630]: ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:39:18.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:18 smithi110 ceph-mon[27365]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:39:18.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:18 smithi110 ceph-mon[27365]: pgmap v30: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 31 op/s 2024-09-14T03:39:18.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:18 smithi195 ceph-mon[30695]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:39:18.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:18 smithi195 ceph-mon[30695]: pgmap v30: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 31 op/s 2024-09-14T03:39:18.877 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:18 smithi110 podman[77642]: 2024-09-14 03:39:18.502640994 +0000 UTC m=+0.694244185 container remove 764997e49f35d23edf747b6ce67f2dab482d406113349b9f5709722242ad63be (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:18.877 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:18 smithi110 podman[77642]: 2024-09-14 03:39:18.670416284 +0000 UTC m=+0.862019438 volume remove e776c50bf85ba8dd69f38367de98d766fe91dd0d640dd7c0bfa25ba6e0e69030 2024-09-14T03:39:19.047 INFO:tasks.rados.rados.0.smithi110.stdout:577: done (1 left) 2024-09-14T03:39:19.047 INFO:tasks.rados.rados.0.smithi110.stdout:578: done (0 left) 2024-09-14T03:39:19.048 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 57 2024-09-14T03:39:19.048 INFO:tasks.rados.rados.0.smithi110.stdout:579: seq_num 198 ranges {0=524288,524288=401408} 2024-09-14T03:39:19.050 INFO:tasks.rados.rados.0.smithi110.stdout:579: writing smithi11075562-27 from 0 to 524288 tid 1 2024-09-14T03:39:19.052 INFO:tasks.rados.rados.0.smithi110.stdout:579: writing smithi11075562-27 from 524288 to 925696 tid 2 2024-09-14T03:39:19.052 INFO:tasks.rados.rados.0.smithi110.stdout:580: rollback oid 16 current snap is 57 2024-09-14T03:39:19.052 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 56 2024-09-14T03:39:19.053 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 57 2024-09-14T03:39:19.053 INFO:tasks.rados.rados.0.smithi110.stdout:581: seq_num 199 ranges {2441216=540672,2981888=401408} 2024-09-14T03:39:19.069 INFO:tasks.rados.rados.0.smithi110.stdout:581: writing smithi11075562-32 from 2441216 to 2981888 tid 1 2024-09-14T03:39:19.071 INFO:tasks.rados.rados.0.smithi110.stdout:581: writing smithi11075562-32 from 2981888 to 3383296 tid 2 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:582: snap_create 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:579: finishing write tid 1 to smithi11075562-27 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:580: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 187 (ObjNum 81 snap 0 seq_num 64605168) dirty dne 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:579: finishing write tid 2 to smithi11075562-27 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:579: finishing write tid 3 to smithi11075562-27 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:579: finishing write tid 4 to smithi11075562-27 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 88 (ObjNum 198 snap 57 seq_num 198) dirty exists 2024-09-14T03:39:19.072 INFO:tasks.rados.rados.0.smithi110.stdout:579: left oid 27 (ObjNum 198 snap 57 seq_num 198) 2024-09-14T03:39:19.073 INFO:tasks.rados.rados.0.smithi110.stdout:581: finishing write tid 1 to smithi11075562-32 2024-09-14T03:39:19.073 INFO:tasks.rados.rados.0.smithi110.stdout:581: finishing write tid 2 to smithi11075562-32 2024-09-14T03:39:19.073 INFO:tasks.rados.rados.0.smithi110.stdout:581: finishing write tid 3 to smithi11075562-32 2024-09-14T03:39:19.074 INFO:tasks.rados.rados.0.smithi110.stdout:581: finishing write tid 4 to smithi11075562-32 2024-09-14T03:39:19.074 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 185 (ObjNum 199 snap 57 seq_num 199) dirty exists 2024-09-14T03:39:19.074 INFO:tasks.rados.rados.0.smithi110.stdout:581: left oid 32 (ObjNum 199 snap 57 seq_num 199) 2024-09-14T03:39:19.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-mon[35374]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:39:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-mon[27365]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:39:19.197 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:18 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:39:19.197 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:18 smithi110 systemd[1]: Stopped Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:19.198 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 systemd[1]: Starting Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:19 smithi195 ceph-mon[30695]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:39:19.608 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 podman[77770]: 2024-09-14 03:39:19.208318999 +0000 UTC m=+0.016295196 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-14T03:39:19.609 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 podman[77770]: 2024-09-14 03:39:19.32140179 +0000 UTC m=+0.129377972 volume create 37e69e81b5357465d40bcac9d03dc22b7570e99c74a10859c0a5a3fec87d0a8c 2024-09-14T03:39:19.609 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 podman[77770]: 2024-09-14 03:39:19.405075572 +0000 UTC m=+0.213051754 container create 02afc2b875a4e86e3e7d582969888ef499c384223706a4408acb467431fa6003 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:19.946 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 podman[77770]: 2024-09-14 03:39:19.708338835 +0000 UTC m=+0.516315022 container init 02afc2b875a4e86e3e7d582969888ef499c384223706a4408acb467431fa6003 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 podman[77770]: 2024-09-14 03:39:19.711325545 +0000 UTC m=+0.519301730 container start 02afc2b875a4e86e3e7d582969888ef499c384223706a4408acb467431fa6003 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:19.722Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:19.722Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:19.723Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.110 port=9094 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:19.724Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:19.746Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:19.746Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:19.748Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:19.748Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 bash[77770]: 02afc2b875a4e86e3e7d582969888ef499c384223706a4408acb467431fa6003 2024-09-14T03:39:19.947 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:19 smithi110 systemd[1]: Started Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:20.032 INFO:tasks.rados.rados.0.smithi110.stdout:579: done (3 left) 2024-09-14T03:39:20.032 INFO:tasks.rados.rados.0.smithi110.stdout:580: done (2 left) 2024-09-14T03:39:20.032 INFO:tasks.rados.rados.0.smithi110.stdout:581: done (1 left) 2024-09-14T03:39:20.032 INFO:tasks.rados.rados.0.smithi110.stdout:582: done (0 left) 2024-09-14T03:39:20.032 INFO:tasks.rados.rados.0.smithi110.stdout:583: setattr oid 7 current snap is 58 2024-09-14T03:39:20.033 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 58 2024-09-14T03:39:20.033 INFO:tasks.rados.rados.0.smithi110.stdout:584: seq_num 200 ranges {2441216=557056,2998272=401408} 2024-09-14T03:39:20.049 INFO:tasks.rados.rados.0.smithi110.stdout:584: writing smithi11075562-19 from 2441216 to 2998272 tid 1 2024-09-14T03:39:20.052 INFO:tasks.rados.rados.0.smithi110.stdout:584: writing smithi11075562-19 from 2998272 to 3399680 tid 2 2024-09-14T03:39:20.052 INFO:tasks.rados.rados.0.smithi110.stdout:585: copy_from oid 13 from oid 8 current snap is 58 2024-09-14T03:39:20.052 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 101 (ObjNum 154 snap 45 seq_num 154) dirty exists 2024-09-14T03:39:20.052 INFO:tasks.rados.rados.0.smithi110.stdout:583: done (2 left) 2024-09-14T03:39:20.053 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 58 2024-09-14T03:39:20.053 INFO:tasks.rados.rados.0.smithi110.stdout:586: seq_num 201 ranges {303104=573440,876544=401408} 2024-09-14T03:39:20.057 INFO:tasks.rados.rados.0.smithi110.stdout:586: writing smithi11075562-41 from 303104 to 876544 tid 1 2024-09-14T03:39:20.060 INFO:tasks.rados.rados.0.smithi110.stdout:586: writing smithi11075562-41 from 876544 to 1277952 tid 2 2024-09-14T03:39:20.060 INFO:tasks.rados.rados.0.smithi110.stdout:587: read oid 20 snap 20 2024-09-14T03:39:20.060 INFO:tasks.rados.rados.0.smithi110.stdout:587: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-14T03:39:20.060 INFO:tasks.rados.rados.0.smithi110.stdout:588: read oid 21 snap -1 2024-09-14T03:39:20.060 INFO:tasks.rados.rados.0.smithi110.stdout:588: expect deleted 2024-09-14T03:39:20.060 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 58 2024-09-14T03:39:20.060 INFO:tasks.rados.rados.0.smithi110.stdout:589: seq_num 202 ranges {4128768=589824,4718592=401408} 2024-09-14T03:39:20.086 INFO:tasks.rados.rados.0.smithi110.stdout:589: writing smithi11075562-26 from 4128768 to 4718592 tid 1 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:589: writing smithi11075562-26 from 4718592 to 5120000 tid 2 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:590: rollback oid 12 current snap is 58 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 21 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:584: finishing write tid 1 to smithi11075562-19 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:584: finishing write tid 2 to smithi11075562-19 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:584: finishing write tid 3 to smithi11075562-19 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:584: finishing write tid 4 to smithi11075562-19 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 222 (ObjNum 200 snap 58 seq_num 200) dirty exists 2024-09-14T03:39:20.089 INFO:tasks.rados.rados.0.smithi110.stdout:584: left oid 19 (ObjNum 200 snap 58 seq_num 200) 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:586: finishing write tid 1 to smithi11075562-41 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:584: done (6 left) 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:591: rollback oid 31 current snap is 58 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 44 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:586: finishing write tid 2 to smithi11075562-41 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:586: finishing write tid 3 to smithi11075562-41 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:586: finishing write tid 4 to smithi11075562-41 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 252 (ObjNum 201 snap 58 seq_num 201) dirty exists 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:586: left oid 41 (ObjNum 201 snap 58 seq_num 201) 2024-09-14T03:39:20.090 INFO:tasks.rados.rados.0.smithi110.stdout:587: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-14T03:39:20.111 INFO:tasks.rados.rados.0.smithi110.stdout:585: finishing copy_from racing read to smithi11075562-13 2024-09-14T03:39:20.111 INFO:tasks.rados.rados.0.smithi110.stdout:585: finishing copy_from to smithi11075562-13 2024-09-14T03:39:20.111 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 113 (ObjNum 135 snap 37 seq_num 135) dirty exists 2024-09-14T03:39:20.111 INFO:tasks.rados.rados.0.smithi110.stdout:585: done (6 left) 2024-09-14T03:39:20.111 INFO:tasks.rados.rados.0.smithi110.stdout:586: done (5 left) 2024-09-14T03:39:20.111 INFO:tasks.rados.rados.0.smithi110.stdout:587: done (4 left) 2024-09-14T03:39:20.111 INFO:tasks.rados.rados.0.smithi110.stdout:588: done (3 left) 2024-09-14T03:39:20.112 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 58 2024-09-14T03:39:20.112 INFO:tasks.rados.rados.0.smithi110.stdout:591: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:39:20.112 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 64 (ObjNum 145 snap 0 seq_num 64899984) dirty dne 2024-09-14T03:39:20.112 INFO:tasks.rados.rados.0.smithi110.stdout:589: finishing write tid 1 to smithi11075562-26 2024-09-14T03:39:20.112 INFO:tasks.rados.rados.0.smithi110.stdout:589: finishing write tid 2 to smithi11075562-26 2024-09-14T03:39:20.112 INFO:tasks.rados.rados.0.smithi110.stdout:592: seq_num 203 ranges {2179072=606208,2785280=401408} 2024-09-14T03:39:20.126 INFO:tasks.rados.rados.0.smithi110.stdout:592: writing smithi11075562-33 from 2179072 to 2785280 tid 1 2024-09-14T03:39:20.129 INFO:tasks.rados.rados.0.smithi110.stdout:592: writing smithi11075562-33 from 2785280 to 3186688 tid 2 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:591: done (3 left) 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:593: rollback oid 50 current snap is 58 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 44 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:589: finishing write tid 3 to smithi11075562-26 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:590: finishing rollback tid 0 to smithi11075562-12 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:590: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 299 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:589: finishing write tid 4 to smithi11075562-26 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 125 (ObjNum 202 snap 58 seq_num 202) dirty exists 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:589: left oid 26 (ObjNum 202 snap 58 seq_num 202) 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:589: done (3 left) 2024-09-14T03:39:20.130 INFO:tasks.rados.rados.0.smithi110.stdout:590: done (2 left) 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:594: read oid 43 snap -1 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:594: expect deleted 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:595: rollback oid 28 current snap is 58 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 28 to 53 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:596: read oid 8 snap -1 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:596: expect (ObjNum 135 snap 37 seq_num 135) 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:597: rollback oid 34 current snap is 58 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 53 2024-09-14T03:39:20.131 INFO:tasks.rados.rados.0.smithi110.stdout:598: snap_create 2024-09-14T03:39:20.132 INFO:tasks.rados.rados.0.smithi110.stdout:595: finishing rollback tid 0 to smithi11075562-28 2024-09-14T03:39:20.132 INFO:tasks.rados.rados.0.smithi110.stdout:597: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:39:20.132 INFO:tasks.rados.rados.0.smithi110.stdout:592: finishing write tid 1 to smithi11075562-33 2024-09-14T03:39:20.132 INFO:tasks.rados.rados.0.smithi110.stdout:595: finishing rollback tid 1 to smithi11075562-28 2024-09-14T03:39:20.132 INFO:tasks.rados.rados.0.smithi110.stdout:593: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:39:20.132 INFO:tasks.rados.rados.0.smithi110.stdout:592: finishing write tid 2 to smithi11075562-33 2024-09-14T03:39:20.133 INFO:tasks.rados.rados.0.smithi110.stdout:595: finishing rollback tid 2 to smithi11075562-28 2024-09-14T03:39:20.133 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 165 (ObjNum 168 snap 48 seq_num 168) dirty exists 2024-09-14T03:39:20.134 INFO:tasks.rados.rados.0.smithi110.stdout:597: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:39:20.134 INFO:tasks.rados.rados.0.smithi110.stdout:593: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:39:20.134 INFO:tasks.rados.rados.0.smithi110.stdout:597: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:39:20.134 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 71 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:39:20.135 INFO:tasks.rados.rados.0.smithi110.stdout:593: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:39:20.135 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 40 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:39:20.135 INFO:tasks.rados.rados.0.smithi110.stdout:592: finishing write tid 3 to smithi11075562-33 2024-09-14T03:39:20.135 INFO:tasks.rados.rados.0.smithi110.stdout:592: finishing write tid 4 to smithi11075562-33 2024-09-14T03:39:20.135 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 176 (ObjNum 203 snap 58 seq_num 203) dirty exists 2024-09-14T03:39:20.135 INFO:tasks.rados.rados.0.smithi110.stdout:592: left oid 33 (ObjNum 203 snap 58 seq_num 203) 2024-09-14T03:39:20.136 INFO:tasks.rados.rados.0.smithi110.stdout:596: expect (ObjNum 135 snap 37 seq_num 135) 2024-09-14T03:39:20.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[35374]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:39:20.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[35374]: pgmap v33: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:20.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:20.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:20.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[35374]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:39:20.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[27365]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:39:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[27365]: pgmap v33: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-mon[27365]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:39:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:39:20] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2024-09-14T03:39:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:20 smithi195 ceph-mon[30695]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:39:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:20 smithi195 ceph-mon[30695]: pgmap v33: 121 pgs: 121 active+clean; 136 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:20 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:20 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:20 smithi195 ceph-mon[30695]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:39:20.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:39:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:39:20.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:20 smithi110 systemd[1]: Stopping Ceph node-exporter.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:20.991 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:20 smithi110 podman[77917]: 2024-09-14 03:39:20.708452019 +0000 UTC m=+0.163229238 container died 986afca49027ee03b55ff104e680d54c34410fc4ff196bda718e48370db38706 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:592: done (6 left) 2024-09-14T03:39:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:593: done (5 left) 2024-09-14T03:39:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:594: done (4 left) 2024-09-14T03:39:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:595: done (3 left) 2024-09-14T03:39:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:596: done (2 left) 2024-09-14T03:39:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:597: done (1 left) 2024-09-14T03:39:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:598: done (0 left) 2024-09-14T03:39:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:599: rollback oid 42 current snap is 59 2024-09-14T03:39:21.057 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 53 2024-09-14T03:39:21.057 INFO:tasks.rados.rados.0.smithi110.stdout:600: copy_from oid 3 from oid 1 current snap is 59 2024-09-14T03:39:21.057 INFO:tasks.rados.rados.0.smithi110.stdout:601: snap_remove snap 45 2024-09-14T03:39:21.442 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:21 smithi110 ceph-mon[35374]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-14T03:39:21.442 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:21 smithi110 ceph-mon[35374]: Deploying daemon node-exporter.a on smithi110 2024-09-14T03:39:21.442 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:21 smithi110 podman[77917]: 2024-09-14 03:39:21.173044725 +0000 UTC m=+0.627821943 container cleanup 986afca49027ee03b55ff104e680d54c34410fc4ff196bda718e48370db38706 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:21.442 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:21 smithi110 bash[77917]: ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a 2024-09-14T03:39:21.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:21 smithi110 ceph-mon[27365]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-14T03:39:21.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:21 smithi110 ceph-mon[27365]: Deploying daemon node-exporter.a on smithi110 2024-09-14T03:39:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:21 smithi195 ceph-mon[30695]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-14T03:39:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:21 smithi195 ceph-mon[30695]: Deploying daemon node-exporter.a on smithi110 2024-09-14T03:39:21.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:21 smithi110 podman[77929]: 2024-09-14 03:39:21.441542182 +0000 UTC m=+0.731973667 container remove 986afca49027ee03b55ff104e680d54c34410fc4ff196bda718e48370db38706 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:21.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:21 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:39:21.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:21 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-14T03:39:21.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:21 smithi110 systemd[1]: Stopped Ceph node-exporter.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:21.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:21 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 2.056s CPU time. 2024-09-14T03:39:22.006 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:21 smithi110 systemd[1]: Starting Ceph node-exporter.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:22.006 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:21.725Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000251007s 2024-09-14T03:39:22.070 INFO:tasks.rados.rados.0.smithi110.stdout:601: done (2 left) 2024-09-14T03:39:22.070 INFO:tasks.rados.rados.0.smithi110.stdout:602: snap_create 2024-09-14T03:39:22.070 INFO:tasks.rados.rados.0.smithi110.stdout:599: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:39:22.071 INFO:tasks.rados.rados.0.smithi110.stdout:599: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:39:22.071 INFO:tasks.rados.rados.0.smithi110.stdout:599: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:39:22.071 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 238 (ObjNum 180 snap 51 seq_num 180) dirty exists 2024-09-14T03:39:22.071 INFO:tasks.rados.rados.0.smithi110.stdout:600: finishing copy_from racing read to smithi11075562-3 2024-09-14T03:39:22.071 INFO:tasks.rados.rados.0.smithi110.stdout:600: finishing copy_from to smithi11075562-3 2024-09-14T03:39:22.071 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 174 (ObjNum 115 snap 28 seq_num 115) dirty exists 2024-09-14T03:39:22.341 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:22 smithi110 bash[78061]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-14T03:39:22.342 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:22 smithi110 ceph-mon[35374]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:39:22.342 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:22 smithi110 ceph-mon[35374]: pgmap v36: 121 pgs: 121 active+clean; 146 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-14T03:39:22.342 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:22 smithi110 ceph-mon[27365]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:39:22.342 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:22 smithi110 ceph-mon[27365]: pgmap v36: 121 pgs: 121 active+clean; 146 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-14T03:39:22.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:22 smithi195 ceph-mon[30695]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:39:22.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:22 smithi195 ceph-mon[30695]: pgmap v36: 121 pgs: 121 active+clean; 146 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-14T03:39:22.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:22 smithi110 bash[78061]: Getting image source signatures 2024-09-14T03:39:22.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:22 smithi110 bash[78061]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-14T03:39:22.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:22 smithi110 bash[78061]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-14T03:39:22.696 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:22 smithi110 bash[78061]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-14T03:39:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:599: done (2 left) 2024-09-14T03:39:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:600: done (1 left) 2024-09-14T03:39:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:602: done (0 left) 2024-09-14T03:39:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:603: read oid 46 snap -1 2024-09-14T03:39:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:603: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:39:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:604: read oid 6 snap -1 2024-09-14T03:39:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:604: expect (ObjNum 176 snap 50 seq_num 176) 2024-09-14T03:39:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:605: rollback oid 31 current snap is 60 2024-09-14T03:39:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 57 2024-09-14T03:39:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:606: delete oid 50 current snap is 60 2024-09-14T03:39:23.072 INFO:tasks.rados.rados.0.smithi110.stdout:605: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:39:23.072 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 64 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:39:23.072 INFO:tasks.rados.rados.0.smithi110.stdout:605: done (3 left) 2024-09-14T03:39:23.072 INFO:tasks.rados.rados.0.smithi110.stdout:606: done (2 left) 2024-09-14T03:39:23.072 INFO:tasks.rados.rados.0.smithi110.stdout:607: read oid 10 snap -1 2024-09-14T03:39:23.072 INFO:tasks.rados.rados.0.smithi110.stdout:607: expect deleted 2024-09-14T03:39:23.072 INFO:tasks.rados.rados.0.smithi110.stdout:608: snap_create 2024-09-14T03:39:23.075 INFO:tasks.rados.rados.0.smithi110.stdout:603: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:39:23.101 INFO:tasks.rados.rados.0.smithi110.stdout:604: expect (ObjNum 176 snap 50 seq_num 176) 2024-09-14T03:39:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:23 smithi110 ceph-mon[35374]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:39:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:23 smithi110 ceph-mon[27365]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:39:23.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:23 smithi195 ceph-mon[30695]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:39:24.067 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:23 smithi110 bash[78061]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-14T03:39:24.067 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:23 smithi110 bash[78061]: Writing manifest to image destination 2024-09-14T03:39:24.071 INFO:tasks.rados.rados.0.smithi110.stdout:603: done (3 left) 2024-09-14T03:39:24.071 INFO:tasks.rados.rados.0.smithi110.stdout:604: done (2 left) 2024-09-14T03:39:24.071 INFO:tasks.rados.rados.0.smithi110.stdout:607: done (1 left) 2024-09-14T03:39:24.071 INFO:tasks.rados.rados.0.smithi110.stdout:608: done (0 left) 2024-09-14T03:39:24.071 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 61 2024-09-14T03:39:24.071 INFO:tasks.rados.rados.0.smithi110.stdout:609: seq_num 205 ranges {1916928=638976,2555904=401408} 2024-09-14T03:39:24.085 INFO:tasks.rados.rados.0.smithi110.stdout:609: writing smithi11075562-39 from 1916928 to 2555904 tid 1 2024-09-14T03:39:24.087 INFO:tasks.rados.rados.0.smithi110.stdout:609: writing smithi11075562-39 from 2555904 to 2957312 tid 2 2024-09-14T03:39:24.087 INFO:tasks.rados.rados.0.smithi110.stdout:610: read oid 48 snap -1 2024-09-14T03:39:24.087 INFO:tasks.rados.rados.0.smithi110.stdout:610: expect deleted 2024-09-14T03:39:24.087 INFO:tasks.rados.rados.0.smithi110.stdout:611: rollback oid 18 current snap is 61 2024-09-14T03:39:24.087 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 18 to 52 2024-09-14T03:39:24.088 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 61 2024-09-14T03:39:24.088 INFO:tasks.rados.rados.0.smithi110.stdout:612: seq_num 206 ranges {319488=655360,974848=401408} 2024-09-14T03:39:24.093 INFO:tasks.rados.rados.0.smithi110.stdout:612: writing smithi11075562-44 from 319488 to 974848 tid 1 2024-09-14T03:39:24.095 INFO:tasks.rados.rados.0.smithi110.stdout:612: writing smithi11075562-44 from 974848 to 1376256 tid 2 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:613: snap_create 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:611: finishing rollback tid 1 to smithi11075562-18 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 189 (ObjNum 33 snap 0 seq_num 64790832) dirty dne 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:609: finishing write tid 1 to smithi11075562-39 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:609: finishing write tid 2 to smithi11075562-39 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:609: finishing write tid 3 to smithi11075562-39 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:609: finishing write tid 4 to smithi11075562-39 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 264 (ObjNum 205 snap 61 seq_num 205) dirty exists 2024-09-14T03:39:24.096 INFO:tasks.rados.rados.0.smithi110.stdout:609: left oid 39 (ObjNum 205 snap 61 seq_num 205) 2024-09-14T03:39:24.097 INFO:tasks.rados.rados.0.smithi110.stdout:612: finishing write tid 1 to smithi11075562-44 2024-09-14T03:39:24.098 INFO:tasks.rados.rados.0.smithi110.stdout:612: finishing write tid 2 to smithi11075562-44 2024-09-14T03:39:24.098 INFO:tasks.rados.rados.0.smithi110.stdout:612: finishing write tid 3 to smithi11075562-44 2024-09-14T03:39:24.099 INFO:tasks.rados.rados.0.smithi110.stdout:612: finishing write tid 4 to smithi11075562-44 2024-09-14T03:39:24.099 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 17 (ObjNum 206 snap 61 seq_num 206) dirty exists 2024-09-14T03:39:24.099 INFO:tasks.rados.rados.0.smithi110.stdout:612: left oid 44 (ObjNum 206 snap 61 seq_num 206) 2024-09-14T03:39:24.410 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-mon[27365]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:39:24.410 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-mon[27365]: pgmap v39: 121 pgs: 121 active+clean; 146 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-14T03:39:24.410 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-mon[35374]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:39:24.411 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-mon[35374]: pgmap v39: 121 pgs: 121 active+clean; 146 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-14T03:39:24.411 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 podman[78061]: 2024-09-14 03:39:24.211956396 +0000 UTC m=+2.303037926 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-14T03:39:24.411 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 podman[78061]: 2024-09-14 03:39:24.302089094 +0000 UTC m=+2.393170672 container create 1b0d2ce16092442a3deff043edfd5a3285fbc29a29b98863f3f2d3fa2da08b7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:24.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:24 smithi195 ceph-mon[30695]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:39:24.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:24 smithi195 ceph-mon[30695]: pgmap v39: 121 pgs: 121 active+clean; 146 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-14T03:39:24.946 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 podman[78061]: 2024-09-14 03:39:24.520805111 +0000 UTC m=+2.611886637 container init 1b0d2ce16092442a3deff043edfd5a3285fbc29a29b98863f3f2d3fa2da08b7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:24.946 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 podman[78061]: 2024-09-14 03:39:24.524026976 +0000 UTC m=+2.615108495 container start 1b0d2ce16092442a3deff043edfd5a3285fbc29a29b98863f3f2d3fa2da08b7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:24.946 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.527Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.527Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.527Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.527Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:39:24.947 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:39:24.948 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:39:24.948 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:39:24.948 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:39:24.948 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:39:24.948 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:39:24.948 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:39:24.949 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:39:24.950 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.528Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.529Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a[78146]: ts=2024-09-14T03:39:24.529Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 bash[78061]: 1b0d2ce16092442a3deff043edfd5a3285fbc29a29b98863f3f2d3fa2da08b7c 2024-09-14T03:39:24.951 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:39:24 smithi110 systemd[1]: Started Ceph node-exporter.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:25.033 INFO:tasks.rados.rados.0.smithi110.stdout:609: done (4 left) 2024-09-14T03:39:25.033 INFO:tasks.rados.rados.0.smithi110.stdout:610: done (3 left) 2024-09-14T03:39:25.034 INFO:tasks.rados.rados.0.smithi110.stdout:611: done (2 left) 2024-09-14T03:39:25.034 INFO:tasks.rados.rados.0.smithi110.stdout:612: done (1 left) 2024-09-14T03:39:25.034 INFO:tasks.rados.rados.0.smithi110.stdout:613: done (0 left) 2024-09-14T03:39:25.034 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 62 2024-09-14T03:39:25.034 INFO:tasks.rados.rados.0.smithi110.stdout:614: seq_num 207 ranges {0=671744,671744=401408} 2024-09-14T03:39:25.037 INFO:tasks.rados.rados.0.smithi110.stdout:614: writing smithi11075562-18 from 0 to 671744 tid 1 2024-09-14T03:39:25.039 INFO:tasks.rados.rados.0.smithi110.stdout:614: writing smithi11075562-18 from 671744 to 1073152 tid 2 2024-09-14T03:39:25.039 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 62 2024-09-14T03:39:25.039 INFO:tasks.rados.rados.0.smithi110.stdout:615: seq_num 208 ranges {2957312=688128,3645440=401408} 2024-09-14T03:39:25.058 INFO:tasks.rados.rados.0.smithi110.stdout:615: writing smithi11075562-39 from 2957312 to 3645440 tid 1 2024-09-14T03:39:25.061 INFO:tasks.rados.rados.0.smithi110.stdout:615: writing smithi11075562-39 from 3645440 to 4046848 tid 2 2024-09-14T03:39:25.061 INFO:tasks.rados.rados.0.smithi110.stdout:616: setattr oid 49 current snap is 62 2024-09-14T03:39:25.061 INFO:tasks.rados.rados.0.smithi110.stdout:614: finishing write tid 1 to smithi11075562-18 2024-09-14T03:39:25.061 INFO:tasks.rados.rados.0.smithi110.stdout:614: finishing write tid 2 to smithi11075562-18 2024-09-14T03:39:25.061 INFO:tasks.rados.rados.0.smithi110.stdout:614: finishing write tid 3 to smithi11075562-18 2024-09-14T03:39:25.062 INFO:tasks.rados.rados.0.smithi110.stdout:614: finishing write tid 4 to smithi11075562-18 2024-09-14T03:39:25.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 234 (ObjNum 207 snap 62 seq_num 207) dirty exists 2024-09-14T03:39:25.062 INFO:tasks.rados.rados.0.smithi110.stdout:614: left oid 18 (ObjNum 207 snap 62 seq_num 207) 2024-09-14T03:39:25.062 INFO:tasks.rados.rados.0.smithi110.stdout:614: done (2 left) 2024-09-14T03:39:25.063 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 62 2024-09-14T03:39:25.063 INFO:tasks.rados.rados.0.smithi110.stdout:617: seq_num 209 ranges {2269184=704512,2973696=401408} 2024-09-14T03:39:25.078 INFO:tasks.rados.rados.0.smithi110.stdout:617: writing smithi11075562-8 from 2269184 to 2973696 tid 1 2024-09-14T03:39:25.081 INFO:tasks.rados.rados.0.smithi110.stdout:617: writing smithi11075562-8 from 2973696 to 3375104 tid 2 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:618: rollback oid 23 current snap is 62 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 23 to 57 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:615: finishing write tid 1 to smithi11075562-39 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:619: copy_from oid 9 from oid 18 current snap is 62 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 250 (ObjNum 193 snap 54 seq_num 193) dirty exists 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:615: finishing write tid 2 to smithi11075562-39 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:615: finishing write tid 3 to smithi11075562-39 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:615: finishing write tid 4 to smithi11075562-39 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 268 (ObjNum 208 snap 62 seq_num 208) dirty exists 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:615: left oid 39 (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:615: done (4 left) 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:616: done (3 left) 2024-09-14T03:39:25.082 INFO:tasks.rados.rados.0.smithi110.stdout:620: snap_remove snap 47 2024-09-14T03:39:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:25 smithi110 ceph-mon[27365]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:39:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:25 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:25 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:25 smithi110 ceph-mon[27365]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:39:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:25 smithi110 ceph-mon[35374]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:39:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:25 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:25 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:25 smithi110 ceph-mon[35374]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:39:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:25 smithi195 ceph-mon[30695]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:39:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:25 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:25 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:25 smithi195 ceph-mon[30695]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:39:26.080 INFO:tasks.rados.rados.0.smithi110.stdout:620: done (3 left) 2024-09-14T03:39:26.080 INFO:tasks.rados.rados.0.smithi110.stdout:621: copy_from oid 30 from oid 35 current snap is 62 2024-09-14T03:39:26.080 INFO:tasks.rados.rados.0.smithi110.stdout:618: finishing rollback tid 1 to smithi11075562-23 2024-09-14T03:39:26.081 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 95 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:39:26.081 INFO:tasks.rados.rados.0.smithi110.stdout:617: finishing write tid 1 to smithi11075562-8 2024-09-14T03:39:26.081 INFO:tasks.rados.rados.0.smithi110.stdout:617: finishing write tid 2 to smithi11075562-8 2024-09-14T03:39:26.081 INFO:tasks.rados.rados.0.smithi110.stdout:617: finishing write tid 3 to smithi11075562-8 2024-09-14T03:39:26.081 INFO:tasks.rados.rados.0.smithi110.stdout:617: finishing write tid 4 to smithi11075562-8 2024-09-14T03:39:26.081 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 195 (ObjNum 209 snap 62 seq_num 209) dirty exists 2024-09-14T03:39:26.082 INFO:tasks.rados.rados.0.smithi110.stdout:617: left oid 8 (ObjNum 209 snap 62 seq_num 209) 2024-09-14T03:39:26.082 INFO:tasks.rados.rados.0.smithi110.stdout:619: finishing copy_from racing read to smithi11075562-9 2024-09-14T03:39:26.082 INFO:tasks.rados.rados.0.smithi110.stdout:619: finishing copy_from to smithi11075562-9 2024-09-14T03:39:26.082 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 234 (ObjNum 207 snap 62 seq_num 207) dirty exists 2024-09-14T03:39:26.082 INFO:tasks.rados.rados.0.smithi110.stdout:617: done (3 left) 2024-09-14T03:39:26.083 INFO:tasks.rados.rados.0.smithi110.stdout:618: done (2 left) 2024-09-14T03:39:26.083 INFO:tasks.rados.rados.0.smithi110.stdout:619: done (1 left) 2024-09-14T03:39:26.083 INFO:tasks.rados.rados.0.smithi110.stdout:622: read oid 44 snap 56 2024-09-14T03:39:26.083 INFO:tasks.rados.rados.0.smithi110.stdout:622: expect (ObjNum 162 snap 47 seq_num 162) 2024-09-14T03:39:26.083 INFO:tasks.rados.rados.0.smithi110.stdout:623: copy_from oid 46 from oid 10 current snap is 62 2024-09-14T03:39:26.084 INFO:tasks.rados.rados.0.smithi110.stdout:624: delete oid 19 current snap is 62 2024-09-14T03:39:26.085 INFO:tasks.rados.rados.0.smithi110.stdout:623: finishing copy_from to smithi11075562-46 2024-09-14T03:39:26.085 INFO:tasks.rados.rados.0.smithi110.stdout:623: got expected ENOENT (src dne) 2024-09-14T03:39:26.085 INFO:tasks.rados.rados.0.smithi110.stdout:623: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:39:26.090 INFO:tasks.rados.rados.0.smithi110.stdout:622: expect (ObjNum 162 snap 47 seq_num 162) 2024-09-14T03:39:26.101 INFO:tasks.rados.rados.0.smithi110.stdout:621: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:39:26.101 INFO:tasks.rados.rados.0.smithi110.stdout:621: finishing copy_from to smithi11075562-30 2024-09-14T03:39:26.102 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 192 (ObjNum 186 snap 51 seq_num 186) dirty exists 2024-09-14T03:39:26.102 INFO:tasks.rados.rados.0.smithi110.stdout:621: done (3 left) 2024-09-14T03:39:26.102 INFO:tasks.rados.rados.0.smithi110.stdout:622: done (2 left) 2024-09-14T03:39:26.102 INFO:tasks.rados.rados.0.smithi110.stdout:623: done (1 left) 2024-09-14T03:39:26.102 INFO:tasks.rados.rados.0.smithi110.stdout:624: done (0 left) 2024-09-14T03:39:26.102 INFO:tasks.rados.rados.0.smithi110.stdout:625: snap_create 2024-09-14T03:39:26.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:26 smithi110 ceph-mon[35374]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:39:26.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:26 smithi110 ceph-mon[35374]: Deploying daemon prometheus.a on smithi195 2024-09-14T03:39:26.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:26 smithi110 ceph-mon[35374]: pgmap v42: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:39:26.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:26 smithi110 ceph-mon[27365]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:39:26.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:26 smithi110 ceph-mon[27365]: Deploying daemon prometheus.a on smithi195 2024-09-14T03:39:26.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:26 smithi110 ceph-mon[27365]: pgmap v42: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:39:26.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:26 smithi195 ceph-mon[30695]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:39:26.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:26 smithi195 ceph-mon[30695]: Deploying daemon prometheus.a on smithi195 2024-09-14T03:39:26.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:26 smithi195 ceph-mon[30695]: pgmap v42: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:39:27.082 INFO:tasks.rados.rados.0.smithi110.stdout:625: done (0 left) 2024-09-14T03:39:27.082 INFO:tasks.rados.rados.0.smithi110.stdout:626: read oid 45 snap -1 2024-09-14T03:39:27.082 INFO:tasks.rados.rados.0.smithi110.stdout:626: expect (ObjNum 188 snap 52 seq_num 188) 2024-09-14T03:39:27.082 INFO:tasks.rados.rados.0.smithi110.stdout:627: copy_from oid 49 from oid 4 current snap is 63 2024-09-14T03:39:27.082 INFO:tasks.rados.rados.0.smithi110.stdout:628: snap_remove snap 46 2024-09-14T03:39:27.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:27 smithi110 ceph-mon[27365]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:39:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:27 smithi110 ceph-mon[35374]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:39:27.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:27 smithi195 ceph-mon[30695]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:39:28.090 INFO:tasks.rados.rados.0.smithi110.stdout:628: done (2 left) 2024-09-14T03:39:28.090 INFO:tasks.rados.rados.0.smithi110.stdout:629: rollback oid 14 current snap is 63 2024-09-14T03:39:28.090 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 14 to 61 2024-09-14T03:39:28.090 INFO:tasks.rados.rados.0.smithi110.stdout:627: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:39:28.090 INFO:tasks.rados.rados.0.smithi110.stdout:626: expect (ObjNum 188 snap 52 seq_num 188) 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:627: finishing copy_from to smithi11075562-49 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:629: finishing rollback tid 0 to smithi11075562-14 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:629: finishing rollback tid 1 to smithi11075562-14 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:629: finishing rollback tid 2 to smithi11075562-14 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 280 (ObjNum 184 snap 51 seq_num 184) dirty exists 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:626: done (2 left) 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:627: done (1 left) 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:629: done (0 left) 2024-09-14T03:39:28.117 INFO:tasks.rados.rados.0.smithi110.stdout:630: snap_create 2024-09-14T03:39:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:28 smithi110 ceph-mon[35374]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:39:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:28 smithi110 ceph-mon[35374]: pgmap v45: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:39:28.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:28 smithi110 ceph-mon[27365]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:39:28.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:28 smithi110 ceph-mon[27365]: pgmap v45: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:39:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:28 smithi195 ceph-mon[30695]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:39:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:28 smithi195 ceph-mon[30695]: pgmap v45: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-14T03:39:29.092 INFO:tasks.rados.rados.0.smithi110.stdout:630: done (0 left) 2024-09-14T03:39:29.092 INFO:tasks.rados.rados.0.smithi110.stdout:631: copy_from oid 16 from oid 10 current snap is 64 2024-09-14T03:39:29.093 INFO:tasks.rados.rados.0.smithi110.stdout:632: snap_remove snap 54 2024-09-14T03:39:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:29 smithi110 ceph-mon[27365]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:39:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:29 smithi110 ceph-mon[35374]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:39:29.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:29 smithi195 ceph-mon[30695]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:39:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:632: done (1 left) 2024-09-14T03:39:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:633: read oid 35 snap -1 2024-09-14T03:39:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:633: expect (ObjNum 186 snap 51 seq_num 186) 2024-09-14T03:39:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:634: snap_create 2024-09-14T03:39:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:631: finishing copy_from to smithi11075562-16 2024-09-14T03:39:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:631: got expected ENOENT (src dne) 2024-09-14T03:39:30.036 INFO:tasks.rados.rados.0.smithi110.stdout:631: finishing copy_from racing read to smithi11075562-16 2024-09-14T03:39:30.047 INFO:tasks.rados.rados.0.smithi110.stdout:633: expect (ObjNum 186 snap 51 seq_num 186) 2024-09-14T03:39:30.093 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:39:29 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:39:29.727Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002234534s 2024-09-14T03:39:30.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:30 smithi110 ceph-mon[35374]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:39:30.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:30 smithi110 ceph-mon[35374]: pgmap v48: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:30.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:30 smithi110 ceph-mon[35374]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:39:30.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:30 smithi110 ceph-mon[27365]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:39:30.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:30 smithi110 ceph-mon[27365]: pgmap v48: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:30.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:30 smithi110 ceph-mon[27365]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:39:30.458 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:39:30] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2024-09-14T03:39:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:30 smithi195 ceph-mon[30695]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:39:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:30 smithi195 ceph-mon[30695]: pgmap v48: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:30 smithi195 ceph-mon[30695]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:39:30.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[27595]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:39:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:39:31.036 INFO:tasks.rados.rados.0.smithi110.stdout:631: done (2 left) 2024-09-14T03:39:31.036 INFO:tasks.rados.rados.0.smithi110.stdout:633: done (1 left) 2024-09-14T03:39:31.036 INFO:tasks.rados.rados.0.smithi110.stdout:634: done (0 left) 2024-09-14T03:39:31.036 INFO:tasks.rados.rados.0.smithi110.stdout:635: rmattr oid 33 current snap is 65 2024-09-14T03:39:31.036 INFO:tasks.rados.rados.0.smithi110.stdout:635: done (0 left) 2024-09-14T03:39:31.036 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 65 2024-09-14T03:39:31.036 INFO:tasks.rados.rados.0.smithi110.stdout:636: seq_num 211 ranges {3383296=737280,4120576=401408} 2024-09-14T03:39:31.057 INFO:tasks.rados.rados.0.smithi110.stdout:636: writing smithi11075562-32 from 3383296 to 4120576 tid 1 2024-09-14T03:39:31.059 INFO:tasks.rados.rados.0.smithi110.stdout:636: writing smithi11075562-32 from 4120576 to 4521984 tid 2 2024-09-14T03:39:31.059 INFO:tasks.rados.rados.0.smithi110.stdout:637: copy_from oid 11 from oid 45 current snap is 65 2024-09-14T03:39:31.059 INFO:tasks.rados.rados.0.smithi110.stdout:638: read oid 12 snap -1 2024-09-14T03:39:31.059 INFO:tasks.rados.rados.0.smithi110.stdout:638: expect deleted 2024-09-14T03:39:31.060 INFO:tasks.rados.rados.0.smithi110.stdout:639: rollback oid 13 current snap is 65 2024-09-14T03:39:31.060 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 42 2024-09-14T03:39:31.060 INFO:tasks.rados.rados.0.smithi110.stdout:640: setattr oid 30 current snap is 65 2024-09-14T03:39:31.062 INFO:tasks.rados.rados.0.smithi110.stdout:641: rmattr oid 41 current snap is 65 2024-09-14T03:39:31.062 INFO:tasks.rados.rados.0.smithi110.stdout:642: read oid 31 snap -1 2024-09-14T03:39:31.062 INFO:tasks.rados.rados.0.smithi110.stdout:642: expect deleted 2024-09-14T03:39:31.062 INFO:tasks.rados.rados.0.smithi110.stdout:643: rollback oid 26 current snap is 65 2024-09-14T03:39:31.062 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 48 2024-09-14T03:39:31.062 INFO:tasks.rados.rados.0.smithi110.stdout:644: copy_from oid 23 from oid 1 current snap is 65 2024-09-14T03:39:31.062 INFO:tasks.rados.rados.0.smithi110.stdout:645: delete oid 25 current snap is 65 2024-09-14T03:39:31.065 INFO:tasks.rados.rados.0.smithi110.stdout:636: finishing write tid 1 to smithi11075562-32 2024-09-14T03:39:31.066 INFO:tasks.rados.rados.0.smithi110.stdout:636: finishing write tid 2 to smithi11075562-32 2024-09-14T03:39:31.066 INFO:tasks.rados.rados.0.smithi110.stdout:636: finishing write tid 3 to smithi11075562-32 2024-09-14T03:39:31.067 INFO:tasks.rados.rados.0.smithi110.stdout:639: finishing rollback tid 0 to smithi11075562-13 2024-09-14T03:39:31.067 INFO:tasks.rados.rados.0.smithi110.stdout:639: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:39:31.067 INFO:tasks.rados.rados.0.smithi110.stdout:639: finishing rollback tid 2 to smithi11075562-13 2024-09-14T03:39:31.067 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 278 (ObjNum 140 snap 39 seq_num 140) dirty exists 2024-09-14T03:39:31.068 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 227 (ObjNum 186 snap 51 seq_num 186) dirty exists 2024-09-14T03:39:31.069 INFO:tasks.rados.rados.0.smithi110.stdout:636: finishing write tid 4 to smithi11075562-32 2024-09-14T03:39:31.069 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 225 (ObjNum 211 snap 65 seq_num 211) dirty exists 2024-09-14T03:39:31.069 INFO:tasks.rados.rados.0.smithi110.stdout:636: left oid 32 (ObjNum 211 snap 65 seq_num 211) 2024-09-14T03:39:31.069 INFO:tasks.rados.rados.0.smithi110.stdout:643: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:39:31.069 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 298 (ObjNum 201 snap 58 seq_num 201) dirty exists 2024-09-14T03:39:31.069 INFO:tasks.rados.rados.0.smithi110.stdout:643: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:39:31.070 INFO:tasks.rados.rados.0.smithi110.stdout:643: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:39:31.070 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 128 (ObjNum 126 snap 33 seq_num 126) dirty exists 2024-09-14T03:39:31.070 INFO:tasks.rados.rados.0.smithi110.stdout:637: finishing copy_from racing read to smithi11075562-11 2024-09-14T03:39:31.071 INFO:tasks.rados.rados.0.smithi110.stdout:637: finishing copy_from to smithi11075562-11 2024-09-14T03:39:31.071 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 292 (ObjNum 188 snap 52 seq_num 188) dirty exists 2024-09-14T03:39:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:636: done (9 left) 2024-09-14T03:39:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:637: done (8 left) 2024-09-14T03:39:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:638: done (7 left) 2024-09-14T03:39:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:639: done (6 left) 2024-09-14T03:39:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:640: done (5 left) 2024-09-14T03:39:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:641: done (4 left) 2024-09-14T03:39:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:642: done (3 left) 2024-09-14T03:39:31.072 INFO:tasks.rados.rados.0.smithi110.stdout:643: done (2 left) 2024-09-14T03:39:31.073 INFO:tasks.rados.rados.0.smithi110.stdout:645: done (1 left) 2024-09-14T03:39:31.073 INFO:tasks.rados.rados.0.smithi110.stdout:646: rollback oid 40 current snap is 65 2024-09-14T03:39:31.073 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 40 to 21 2024-09-14T03:39:31.073 INFO:tasks.rados.rados.0.smithi110.stdout:647: snap_remove snap 58 2024-09-14T03:39:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:31 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:31 smithi110 ceph-mon[35374]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:39:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:31 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:31 smithi110 ceph-mon[27365]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:39:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:31 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:31 smithi195 ceph-mon[30695]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:39:32.038 INFO:tasks.rados.rados.0.smithi110.stdout:647: done (2 left) 2024-09-14T03:39:32.038 INFO:tasks.rados.rados.0.smithi110.stdout:648: read oid 44 snap -1 2024-09-14T03:39:32.038 INFO:tasks.rados.rados.0.smithi110.stdout:648: expect (ObjNum 206 snap 61 seq_num 206) 2024-09-14T03:39:32.039 INFO:tasks.rados.rados.0.smithi110.stdout:649: delete oid 2 current snap is 65 2024-09-14T03:39:32.039 INFO:tasks.rados.rados.0.smithi110.stdout:646: finishing rollback tid 0 to smithi11075562-40 2024-09-14T03:39:32.039 INFO:tasks.rados.rados.0.smithi110.stdout:646: finishing rollback tid 1 to smithi11075562-40 2024-09-14T03:39:32.039 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 257 (ObjNum 81 snap 0 seq_num 3338190369) dirty dne 2024-09-14T03:39:32.039 INFO:tasks.rados.rados.0.smithi110.stdout:644: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:39:32.039 INFO:tasks.rados.rados.0.smithi110.stdout:644: finishing copy_from to smithi11075562-23 2024-09-14T03:39:32.039 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 174 (ObjNum 115 snap 28 seq_num 115) dirty exists 2024-09-14T03:39:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:644: done (3 left) 2024-09-14T03:39:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:646: done (2 left) 2024-09-14T03:39:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:649: done (1 left) 2024-09-14T03:39:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:650: rollback oid 30 current snap is 65 2024-09-14T03:39:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 57 2024-09-14T03:39:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 65 2024-09-14T03:39:32.047 INFO:tasks.rados.rados.0.smithi110.stdout:651: seq_num 214 ranges {0=794624,794624=401408} 2024-09-14T03:39:32.053 INFO:tasks.rados.rados.0.smithi110.stdout:651: writing smithi11075562-48 from 0 to 794624 tid 1 2024-09-14T03:39:32.055 INFO:tasks.rados.rados.0.smithi110.stdout:651: writing smithi11075562-48 from 794624 to 1196032 tid 2 2024-09-14T03:39:32.055 INFO:tasks.rados.rados.0.smithi110.stdout:652: rollback oid 42 current snap is 65 2024-09-14T03:39:32.055 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 48 2024-09-14T03:39:32.056 INFO:tasks.rados.rados.0.smithi110.stdout:648: expect (ObjNum 206 snap 61 seq_num 206) 2024-09-14T03:39:32.096 INFO:tasks.rados.rados.0.smithi110.stdout:650: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:39:32.096 INFO:tasks.rados.rados.0.smithi110.stdout:650: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:39:32.096 INFO:tasks.rados.rados.0.smithi110.stdout:648: done (3 left) 2024-09-14T03:39:32.097 INFO:tasks.rados.rados.0.smithi110.stdout:653: snap_remove snap 61 2024-09-14T03:39:32.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:32 smithi110 ceph-mon[35374]: pgmap v51: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 190 KiB/s wr, 5 op/s 2024-09-14T03:39:32.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:32 smithi110 ceph-mon[35374]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:39:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:32 smithi110 ceph-mon[27365]: pgmap v51: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 190 KiB/s wr, 5 op/s 2024-09-14T03:39:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:32 smithi110 ceph-mon[27365]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:39:32.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:32 smithi195 ceph-mon[30695]: pgmap v51: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 190 KiB/s wr, 5 op/s 2024-09-14T03:39:32.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:32 smithi195 ceph-mon[30695]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:39:33.102 INFO:tasks.rados.rados.0.smithi110.stdout:653: done (3 left) 2024-09-14T03:39:33.102 INFO:tasks.rados.rados.0.smithi110.stdout:654: snap_create 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:650: finishing rollback tid 2 to smithi11075562-30 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 234 (ObjNum 93 snap 19 seq_num 93) dirty exists 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:651: finishing write tid 1 to smithi11075562-48 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:651: finishing write tid 2 to smithi11075562-48 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:651: finishing write tid 3 to smithi11075562-48 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:652: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:652: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:652: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 288 (ObjNum 149 snap 44 seq_num 149) dirty exists 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:651: finishing write tid 4 to smithi11075562-48 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 295 (ObjNum 214 snap 65 seq_num 214) dirty exists 2024-09-14T03:39:33.103 INFO:tasks.rados.rados.0.smithi110.stdout:651: left oid 48 (ObjNum 214 snap 65 seq_num 214) 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:650: done (3 left) 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:651: done (2 left) 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:652: done (1 left) 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:654: done (0 left) 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:655: rmattr oid 40 current snap is 66 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:655: done (0 left) 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:656: read oid 10 snap 52 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:656: expect deleted 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:657: read oid 50 snap -1 2024-09-14T03:39:34.107 INFO:tasks.rados.rados.0.smithi110.stdout:657: expect deleted 2024-09-14T03:39:34.108 INFO:tasks.rados.rados.0.smithi110.stdout:658: setattr oid 7 current snap is 66 2024-09-14T03:39:34.109 INFO:tasks.rados.rados.0.smithi110.stdout:656: done (2 left) 2024-09-14T03:39:34.110 INFO:tasks.rados.rados.0.smithi110.stdout:657: done (1 left) 2024-09-14T03:39:34.110 INFO:tasks.rados.rados.0.smithi110.stdout:659: read oid 17 snap 60 2024-09-14T03:39:34.110 INFO:tasks.rados.rados.0.smithi110.stdout:659: expect (ObjNum 74 snap 12 seq_num 74) 2024-09-14T03:39:34.110 INFO:tasks.rados.rados.0.smithi110.stdout:660: copy_from oid 4 from oid 41 current snap is 66 2024-09-14T03:39:34.110 INFO:tasks.rados.rados.0.smithi110.stdout:661: read oid 47 snap -1 2024-09-14T03:39:34.110 INFO:tasks.rados.rados.0.smithi110.stdout:661: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:39:34.110 INFO:tasks.rados.rados.0.smithi110.stdout:662: snap_remove snap 55 2024-09-14T03:39:34.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:34 smithi110 ceph-mon[35374]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:39:34.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:34 smithi110 ceph-mon[35374]: pgmap v54: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 190 KiB/s wr, 5 op/s 2024-09-14T03:39:34.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:34 smithi110 ceph-mon[27365]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:39:34.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:34 smithi110 ceph-mon[27365]: pgmap v54: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 190 KiB/s wr, 5 op/s 2024-09-14T03:39:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:34 smithi195 ceph-mon[30695]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:39:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:34 smithi195 ceph-mon[30695]: pgmap v54: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 190 KiB/s wr, 5 op/s 2024-09-14T03:39:35.037 INFO:tasks.rados.rados.0.smithi110.stdout:662: done (4 left) 2024-09-14T03:39:35.037 INFO:tasks.rados.rados.0.smithi110.stdout:663: snap_remove snap 59 2024-09-14T03:39:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:35 smithi110 ceph-mon[35374]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:39:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:35 smithi110 ceph-mon[35374]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:39:35.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:35 smithi110 ceph-mon[27365]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:39:35.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:35 smithi110 ceph-mon[27365]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:39:35.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:35 smithi195 ceph-mon[30695]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:39:35.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:35 smithi195 ceph-mon[30695]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:39:36.038 INFO:tasks.rados.rados.0.smithi110.stdout:663: done (4 left) 2024-09-14T03:39:36.038 INFO:tasks.rados.rados.0.smithi110.stdout:664: rollback oid 5 current snap is 66 2024-09-14T03:39:36.038 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 5 to 60 2024-09-14T03:39:36.039 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 119 (ObjNum 154 snap 45 seq_num 154) dirty exists 2024-09-14T03:39:36.039 INFO:tasks.rados.rados.0.smithi110.stdout:660: finishing copy_from racing read to smithi11075562-4 2024-09-14T03:39:36.039 INFO:tasks.rados.rados.0.smithi110.stdout:661: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:39:36.088 INFO:tasks.rados.rados.0.smithi110.stdout:660: finishing copy_from to smithi11075562-4 2024-09-14T03:39:36.088 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 298 (ObjNum 201 snap 58 seq_num 201) dirty exists 2024-09-14T03:39:36.089 INFO:tasks.rados.rados.0.smithi110.stdout:659: expect (ObjNum 74 snap 12 seq_num 74) 2024-09-14T03:39:36.126 INFO:tasks.rados.rados.0.smithi110.stdout:664: finishing rollback tid 0 to smithi11075562-5 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:664: finishing rollback tid 1 to smithi11075562-5 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:664: finishing rollback tid 2 to smithi11075562-5 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 337 (ObjNum 191 snap 53 seq_num 191) dirty exists 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:658: done (4 left) 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:659: done (3 left) 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:660: done (2 left) 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:661: done (1 left) 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:664: done (0 left) 2024-09-14T03:39:36.127 INFO:tasks.rados.rados.0.smithi110.stdout:665: delete oid 17 current snap is 66 2024-09-14T03:39:36.128 INFO:tasks.rados.rados.0.smithi110.stdout:665: done (0 left) 2024-09-14T03:39:36.128 INFO:tasks.rados.rados.0.smithi110.stdout:666: read oid 42 snap -1 2024-09-14T03:39:36.128 INFO:tasks.rados.rados.0.smithi110.stdout:666: expect (ObjNum 149 snap 44 seq_num 149) 2024-09-14T03:39:36.128 INFO:tasks.rados.rados.0.smithi110.stdout:667: setattr oid 5 current snap is 66 2024-09-14T03:39:36.130 INFO:tasks.rados.rados.0.smithi110.stdout:668: rmattr oid 26 current snap is 66 2024-09-14T03:39:36.130 INFO:tasks.rados.rados.0.smithi110.stdout:668: done (2 left) 2024-09-14T03:39:36.130 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 66 2024-09-14T03:39:36.130 INFO:tasks.rados.rados.0.smithi110.stdout:669: seq_num 216 ranges {7741440=425984,8167424=409600,8577024=393216} 2024-09-14T03:39:36.173 INFO:tasks.rados.rados.0.smithi110.stdout:669: writing smithi11075562-38 from 7741440 to 8167424 tid 1 2024-09-14T03:39:36.175 INFO:tasks.rados.rados.0.smithi110.stdout:669: writing smithi11075562-38 from 8167424 to 8577024 tid 2 2024-09-14T03:39:36.178 INFO:tasks.rados.rados.0.smithi110.stdout:669: writing smithi11075562-38 from 8577024 to 8970240 tid 3 2024-09-14T03:39:36.178 INFO:tasks.rados.rados.0.smithi110.stdout:670: snap_create 2024-09-14T03:39:36.178 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 340 (ObjNum 191 snap 53 seq_num 191) dirty exists 2024-09-14T03:39:36.178 INFO:tasks.rados.rados.0.smithi110.stdout:666: expect (ObjNum 149 snap 44 seq_num 149) 2024-09-14T03:39:36.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-mon[30695]: pgmap v57: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 22 op/s 2024-09-14T03:39:36.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-mon[30695]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:39:36.210 INFO:tasks.rados.rados.0.smithi110.stdout:669: finishing write tid 1 to smithi11075562-38 2024-09-14T03:39:36.210 INFO:tasks.rados.rados.0.smithi110.stdout:669: finishing write tid 2 to smithi11075562-38 2024-09-14T03:39:36.210 INFO:tasks.rados.rados.0.smithi110.stdout:669: finishing write tid 3 to smithi11075562-38 2024-09-14T03:39:36.210 INFO:tasks.rados.rados.0.smithi110.stdout:669: finishing write tid 4 to smithi11075562-38 2024-09-14T03:39:36.210 INFO:tasks.rados.rados.0.smithi110.stdout:669: finishing write tid 5 to smithi11075562-38 2024-09-14T03:39:36.210 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 130 (ObjNum 216 snap 66 seq_num 216) dirty exists 2024-09-14T03:39:36.210 INFO:tasks.rados.rados.0.smithi110.stdout:669: left oid 38 (ObjNum 216 snap 66 seq_num 216) 2024-09-14T03:39:36.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:36 smithi110 ceph-mon[35374]: pgmap v57: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 22 op/s 2024-09-14T03:39:36.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:36 smithi110 ceph-mon[35374]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:39:36.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:36 smithi110 ceph-mon[27365]: pgmap v57: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 22 op/s 2024-09-14T03:39:36.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:36 smithi110 ceph-mon[27365]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:39:36.514 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 systemd[1]: Stopping Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:39:36.807 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.513Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:39:36.808 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.514Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:39:36.808 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.514Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:39:36.808 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[61883]: ts=2024-09-14T03:39:36.514Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:39:36.808 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 podman[71497]: 2024-09-14 03:39:36.523800174 +0000 UTC m=+0.138907387 container died 4460d604b8f635d23319035366e36f29a7d0d214dd140f1e48a564b451d0ebb9 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:37.113 INFO:tasks.rados.rados.0.smithi110.stdout:666: done (3 left) 2024-09-14T03:39:37.113 INFO:tasks.rados.rados.0.smithi110.stdout:667: done (2 left) 2024-09-14T03:39:37.114 INFO:tasks.rados.rados.0.smithi110.stdout:669: done (1 left) 2024-09-14T03:39:37.114 INFO:tasks.rados.rados.0.smithi110.stdout:670: done (0 left) 2024-09-14T03:39:37.114 INFO:tasks.rados.rados.0.smithi110.stdout:671: snap_remove snap 48 2024-09-14T03:39:37.208 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 podman[71497]: 2024-09-14 03:39:36.98899995 +0000 UTC m=+0.604107164 container cleanup 4460d604b8f635d23319035366e36f29a7d0d214dd140f1e48a564b451d0ebb9 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:37.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:36 smithi195 bash[71497]: ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:39:37.667 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:37 smithi195 podman[71511]: 2024-09-14 03:39:37.312856836 +0000 UTC m=+0.791288978 container remove 4460d604b8f635d23319035366e36f29a7d0d214dd140f1e48a564b451d0ebb9 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:37.667 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:37 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:39:37.667 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:37 smithi195 systemd[1]: Stopped Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:37.668 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:37 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.595s CPU time. 2024-09-14T03:39:37.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:37 smithi195 systemd[1]: Starting Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:37.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:37 smithi195 podman[71643]: 2024-09-14 03:39:37.776993965 +0000 UTC m=+0.017164521 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-14T03:39:37.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:37 smithi195 podman[71643]: 2024-09-14 03:39:37.889339722 +0000 UTC m=+0.129510310 container create acbe21630a9adbbed840dd0a1e5da576ab71c3eadd21cdfae4b05a1b5979fbac (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:38.121 INFO:tasks.rados.rados.0.smithi110.stdout:671: done (0 left) 2024-09-14T03:39:38.121 INFO:tasks.rados.rados.0.smithi110.stdout:672: snap_remove snap 44 2024-09-14T03:39:38.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:38 smithi110 ceph-mon[27365]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:39:38.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:38 smithi110 ceph-mon[27365]: pgmap v60: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 22 op/s 2024-09-14T03:39:38.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:38 smithi110 ceph-mon[35374]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:39:38.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:38 smithi110 ceph-mon[35374]: pgmap v60: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 22 op/s 2024-09-14T03:39:38.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-mon[30695]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:39:38.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-mon[30695]: pgmap v60: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 22 op/s 2024-09-14T03:39:38.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 podman[71643]: 2024-09-14 03:39:38.10015207 +0000 UTC m=+0.340322612 container init acbe21630a9adbbed840dd0a1e5da576ab71c3eadd21cdfae4b05a1b5979fbac (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:38.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 podman[71643]: 2024-09-14 03:39:38.10285994 +0000 UTC m=+0.343030483 container start acbe21630a9adbbed840dd0a1e5da576ab71c3eadd21cdfae4b05a1b5979fbac (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:39:38.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.131Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-14T03:39:38.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.131Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-14T03:39:38.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.131Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi195 (none))" 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.131Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.131Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.134Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.134Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.137Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.137Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.139Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.139Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.072µs 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.139Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.139Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.151Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.151Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-14T03:39:38.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.151Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.983µs wal_replay_duration=12.711843ms wbl_replay_duration=90ns total_replay_duration=12.753072ms 2024-09-14T03:39:38.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.154Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:39:38.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.154Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-14T03:39:38.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.154Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:39:38.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.174Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.644739ms db_storage=816ns remote_storage=1.197µs web_handler=334ns query_engine=470ns scrape=379.295µs scrape_sd=98.427µs notify=10.483µs notify_sd=12.381µs rules=18.812172ms tracing=6.193µs 2024-09-14T03:39:38.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.174Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-14T03:39:38.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:38.174Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:39:38.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 bash[71643]: acbe21630a9adbbed840dd0a1e5da576ab71c3eadd21cdfae4b05a1b5979fbac 2024-09-14T03:39:38.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:38 smithi195 systemd[1]: Started Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:39.121 INFO:tasks.rados.rados.0.smithi110.stdout:672: done (0 left) 2024-09-14T03:39:39.122 INFO:tasks.rados.rados.0.smithi110.stdout:673: rollback oid 12 current snap is 67 2024-09-14T03:39:39.122 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 62 2024-09-14T03:39:39.122 INFO:tasks.rados.rados.0.smithi110.stdout:674: read oid 21 snap -1 2024-09-14T03:39:39.122 INFO:tasks.rados.rados.0.smithi110.stdout:674: expect deleted 2024-09-14T03:39:39.122 INFO:tasks.rados.rados.0.smithi110.stdout:675: snap_remove snap 52 2024-09-14T03:39:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[27365]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:39:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[27365]: Reconfiguring grafana.a (dependencies changed)... 2024-09-14T03:39:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[27365]: Reconfiguring daemon grafana.a on smithi195 2024-09-14T03:39:39.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[35374]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:39:39.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:39.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:39.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[35374]: Reconfiguring grafana.a (dependencies changed)... 2024-09-14T03:39:39.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:39 smithi110 ceph-mon[35374]: Reconfiguring daemon grafana.a on smithi195 2024-09-14T03:39:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:39 smithi195 ceph-mon[30695]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:39:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:39 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:39 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:39 smithi195 ceph-mon[30695]: Reconfiguring grafana.a (dependencies changed)... 2024-09-14T03:39:39.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:39 smithi195 ceph-mon[30695]: Reconfiguring daemon grafana.a on smithi195 2024-09-14T03:39:40.040 INFO:tasks.rados.rados.0.smithi110.stdout:675: done (2 left) 2024-09-14T03:39:40.040 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 67 2024-09-14T03:39:40.040 INFO:tasks.rados.rados.0.smithi110.stdout:676: seq_num 217 ranges {1376256=442368,1818624=696320,2514944=106496} 2024-09-14T03:39:40.051 INFO:tasks.rados.rados.0.smithi110.stdout:676: writing smithi11075562-44 from 1376256 to 1818624 tid 1 2024-09-14T03:39:40.055 INFO:tasks.rados.rados.0.smithi110.stdout:676: writing smithi11075562-44 from 1818624 to 2514944 tid 2 2024-09-14T03:39:40.056 INFO:tasks.rados.rados.0.smithi110.stdout:676: writing smithi11075562-44 from 2514944 to 2621440 tid 3 2024-09-14T03:39:40.056 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 67 2024-09-14T03:39:40.056 INFO:tasks.rados.rados.0.smithi110.stdout:677: seq_num 218 ranges {499712=458752,958464=581632,1540096=221184} 2024-09-14T03:39:40.061 INFO:tasks.rados.rados.0.smithi110.stdout:677: writing smithi11075562-28 from 499712 to 958464 tid 1 2024-09-14T03:39:40.065 INFO:tasks.rados.rados.0.smithi110.stdout:677: writing smithi11075562-28 from 958464 to 1540096 tid 2 2024-09-14T03:39:40.066 INFO:tasks.rados.rados.0.smithi110.stdout:677: writing smithi11075562-28 from 1540096 to 1761280 tid 3 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 67 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:673: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 299 (ObjNum 33 snap 0 seq_num 64983552) dirty dne 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:676: finishing write tid 1 to smithi11075562-44 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:676: finishing write tid 2 to smithi11075562-44 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:676: finishing write tid 3 to smithi11075562-44 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:676: finishing write tid 4 to smithi11075562-44 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:676: finishing write tid 5 to smithi11075562-44 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 36 (ObjNum 217 snap 67 seq_num 217) dirty exists 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:676: left oid 44 (ObjNum 217 snap 67 seq_num 217) 2024-09-14T03:39:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:677: finishing write tid 1 to smithi11075562-28 2024-09-14T03:39:40.068 INFO:tasks.rados.rados.0.smithi110.stdout:678: seq_num 219 ranges {753664=475136,1228800=466944,1695744=335872} 2024-09-14T03:39:40.073 INFO:tasks.rados.rados.0.smithi110.stdout:678: writing smithi11075562-11 from 753664 to 1228800 tid 1 2024-09-14T03:39:40.076 INFO:tasks.rados.rados.0.smithi110.stdout:678: writing smithi11075562-11 from 1228800 to 1695744 tid 2 2024-09-14T03:39:40.078 INFO:tasks.rados.rados.0.smithi110.stdout:678: writing smithi11075562-11 from 1695744 to 2031616 tid 3 2024-09-14T03:39:40.078 INFO:tasks.rados.rados.0.smithi110.stdout:673: done (4 left) 2024-09-14T03:39:40.078 INFO:tasks.rados.rados.0.smithi110.stdout:674: done (3 left) 2024-09-14T03:39:40.078 INFO:tasks.rados.rados.0.smithi110.stdout:676: done (2 left) 2024-09-14T03:39:40.078 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 67 2024-09-14T03:39:40.078 INFO:tasks.rados.rados.0.smithi110.stdout:677: finishing write tid 2 to smithi11075562-28 2024-09-14T03:39:40.079 INFO:tasks.rados.rados.0.smithi110.stdout:677: finishing write tid 3 to smithi11075562-28 2024-09-14T03:39:40.079 INFO:tasks.rados.rados.0.smithi110.stdout:679: seq_num 220 ranges {0=491520,491520=753664,1245184=49152} 2024-09-14T03:39:40.080 INFO:tasks.rados.rados.0.smithi110.stdout:679: writing smithi11075562-2 from 0 to 491520 tid 1 2024-09-14T03:39:40.085 INFO:tasks.rados.rados.0.smithi110.stdout:679: writing smithi11075562-2 from 491520 to 1245184 tid 2 2024-09-14T03:39:40.085 INFO:tasks.rados.rados.0.smithi110.stdout:679: writing smithi11075562-2 from 1245184 to 1294336 tid 3 2024-09-14T03:39:40.086 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 67 2024-09-14T03:39:40.086 INFO:tasks.rados.rados.0.smithi110.stdout:677: finishing write tid 4 to smithi11075562-28 2024-09-14T03:39:40.086 INFO:tasks.rados.rados.0.smithi110.stdout:677: finishing write tid 5 to smithi11075562-28 2024-09-14T03:39:40.086 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 222 (ObjNum 218 snap 67 seq_num 218) dirty exists 2024-09-14T03:39:40.086 INFO:tasks.rados.rados.0.smithi110.stdout:677: left oid 28 (ObjNum 218 snap 67 seq_num 218) 2024-09-14T03:39:40.086 INFO:tasks.rados.rados.0.smithi110.stdout:678: finishing write tid 1 to smithi11075562-11 2024-09-14T03:39:40.086 INFO:tasks.rados.rados.0.smithi110.stdout:680: seq_num 221 ranges {3940352=507904,4448256=688128,5136384=114688} 2024-09-14T03:39:40.109 INFO:tasks.rados.rados.0.smithi110.stdout:680: writing smithi11075562-15 from 3940352 to 4448256 tid 1 2024-09-14T03:39:40.113 INFO:tasks.rados.rados.0.smithi110.stdout:680: writing smithi11075562-15 from 4448256 to 5136384 tid 2 2024-09-14T03:39:40.114 INFO:tasks.rados.rados.0.smithi110.stdout:680: writing smithi11075562-15 from 5136384 to 5251072 tid 3 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:677: done (3 left) 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:681: setattr oid 30 current snap is 67 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:678: finishing write tid 2 to smithi11075562-11 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:678: finishing write tid 3 to smithi11075562-11 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:678: finishing write tid 4 to smithi11075562-11 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:678: finishing write tid 5 to smithi11075562-11 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 300 (ObjNum 219 snap 67 seq_num 219) dirty exists 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:678: left oid 11 (ObjNum 219 snap 67 seq_num 219) 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:679: finishing write tid 1 to smithi11075562-2 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:679: finishing write tid 2 to smithi11075562-2 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:679: finishing write tid 3 to smithi11075562-2 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:679: finishing write tid 4 to smithi11075562-2 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:679: finishing write tid 5 to smithi11075562-2 2024-09-14T03:39:40.115 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 226 (ObjNum 220 snap 67 seq_num 220) dirty exists 2024-09-14T03:39:40.116 INFO:tasks.rados.rados.0.smithi110.stdout:679: left oid 2 (ObjNum 220 snap 67 seq_num 220) 2024-09-14T03:39:40.116 INFO:tasks.rados.rados.0.smithi110.stdout:680: finishing write tid 1 to smithi11075562-15 2024-09-14T03:39:40.117 INFO:tasks.rados.rados.0.smithi110.stdout:678: done (3 left) 2024-09-14T03:39:40.117 INFO:tasks.rados.rados.0.smithi110.stdout:679: done (2 left) 2024-09-14T03:39:40.117 INFO:tasks.rados.rados.0.smithi110.stdout:682: delete oid 7 current snap is 67 2024-09-14T03:39:40.117 INFO:tasks.rados.rados.0.smithi110.stdout:680: finishing write tid 2 to smithi11075562-15 2024-09-14T03:39:40.117 INFO:tasks.rados.rados.0.smithi110.stdout:680: finishing write tid 3 to smithi11075562-15 2024-09-14T03:39:40.118 INFO:tasks.rados.rados.0.smithi110.stdout:680: finishing write tid 4 to smithi11075562-15 2024-09-14T03:39:40.119 INFO:tasks.rados.rados.0.smithi110.stdout:682: done (2 left) 2024-09-14T03:39:40.120 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 67 2024-09-14T03:39:40.120 INFO:tasks.rados.rados.0.smithi110.stdout:683: seq_num 223 ranges {0=540672,540672=450560,991232=352256} 2024-09-14T03:39:40.123 INFO:tasks.rados.rados.0.smithi110.stdout:683: writing smithi11075562-25 from 0 to 540672 tid 1 2024-09-14T03:39:40.126 INFO:tasks.rados.rados.0.smithi110.stdout:683: writing smithi11075562-25 from 540672 to 991232 tid 2 2024-09-14T03:39:40.129 INFO:tasks.rados.rados.0.smithi110.stdout:683: writing smithi11075562-25 from 991232 to 1343488 tid 3 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:684: copy_from oid 45 from oid 6 current snap is 67 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:680: finishing write tid 5 to smithi11075562-15 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 283 (ObjNum 221 snap 67 seq_num 221) dirty exists 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:680: left oid 15 (ObjNum 221 snap 67 seq_num 221) 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:680: done (3 left) 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:685: rollback oid 15 current snap is 67 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 65 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 285 (ObjNum 93 snap 19 seq_num 93) dirty exists 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:683: finishing write tid 1 to smithi11075562-25 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:681: done (3 left) 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:686: copy_from oid 33 from oid 19 current snap is 67 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:687: copy_from oid 18 from oid 8 current snap is 67 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:683: finishing write tid 2 to smithi11075562-25 2024-09-14T03:39:40.130 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 67 2024-09-14T03:39:40.131 INFO:tasks.rados.rados.0.smithi110.stdout:688: seq_num 224 ranges {0=557056,557056=737280,1294336=65536} 2024-09-14T03:39:40.132 INFO:tasks.rados.rados.0.smithi110.stdout:688: writing smithi11075562-17 from 0 to 557056 tid 1 2024-09-14T03:39:40.137 INFO:tasks.rados.rados.0.smithi110.stdout:688: writing smithi11075562-17 from 557056 to 1294336 tid 2 2024-09-14T03:39:40.137 INFO:tasks.rados.rados.0.smithi110.stdout:688: writing smithi11075562-17 from 1294336 to 1359872 tid 3 2024-09-14T03:39:40.137 INFO:tasks.rados.rados.0.smithi110.stdout:689: snap_remove snap 42 2024-09-14T03:39:40.265 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:40 smithi195 ceph-mon[30695]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:39:40.265 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:40 smithi195 ceph-mon[30695]: pgmap v63: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:40.265 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:40 smithi195 ceph-mon[30695]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:39:40.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:40 smithi110 ceph-mon[35374]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:39:40.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:40 smithi110 ceph-mon[35374]: pgmap v63: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:40.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:40 smithi110 ceph-mon[35374]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:39:40.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:40 smithi110 ceph-mon[27365]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:39:40.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:40 smithi110 ceph-mon[27365]: pgmap v63: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:40.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:40 smithi110 ceph-mon[27365]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:39:41.134 INFO:tasks.rados.rados.0.smithi110.stdout:689: done (6 left) 2024-09-14T03:39:41.134 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 67 2024-09-14T03:39:41.134 INFO:tasks.rados.rados.0.smithi110.stdout:690: seq_num 225 ranges {1761280=573440,2334720=622592,2957312=180224} 2024-09-14T03:39:41.147 INFO:tasks.rados.rados.0.smithi110.stdout:690: writing smithi11075562-28 from 1761280 to 2334720 tid 1 2024-09-14T03:39:41.156 INFO:tasks.rados.rados.0.smithi110.stdout:690: writing smithi11075562-28 from 2334720 to 2957312 tid 2 2024-09-14T03:39:41.156 INFO:tasks.rados.rados.0.smithi110.stdout:690: writing smithi11075562-28 from 2957312 to 3137536 tid 3 2024-09-14T03:39:41.156 INFO:tasks.rados.rados.0.smithi110.stdout:691: read oid 35 snap -1 2024-09-14T03:39:41.156 INFO:tasks.rados.rados.0.smithi110.stdout:691: expect (ObjNum 186 snap 51 seq_num 186) 2024-09-14T03:39:41.156 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 67 2024-09-14T03:39:41.157 INFO:tasks.rados.rados.0.smithi110.stdout:686: finishing copy_from to smithi11075562-33 2024-09-14T03:39:41.157 INFO:tasks.rados.rados.0.smithi110.stdout:686: got expected ENOENT (src dne) 2024-09-14T03:39:41.157 INFO:tasks.rados.rados.0.smithi110.stdout:686: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:39:41.157 INFO:tasks.rados.rados.0.smithi110.stdout:683: finishing write tid 3 to smithi11075562-25 2024-09-14T03:39:41.157 INFO:tasks.rados.rados.0.smithi110.stdout:692: seq_num 226 ranges {2056192=589824,2646016=507904,3153920=294912} 2024-09-14T03:39:41.166 INFO:tasks.rados.rados.0.smithi110.stdout:692: writing smithi11075562-30 from 2056192 to 2646016 tid 1 2024-09-14T03:39:41.170 INFO:tasks.rados.rados.0.smithi110.stdout:692: writing smithi11075562-30 from 2646016 to 3153920 tid 2 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:692: writing smithi11075562-30 from 3153920 to 3448832 tid 3 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:686: done (8 left) 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:693: snap_create 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:683: finishing write tid 4 to smithi11075562-25 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:685: finishing rollback tid 0 to smithi11075562-15 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:685: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:685: finishing rollback tid 2 to smithi11075562-15 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 288 (ObjNum 167 snap 48 seq_num 167) dirty exists 2024-09-14T03:39:41.173 INFO:tasks.rados.rados.0.smithi110.stdout:683: finishing write tid 5 to smithi11075562-25 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 363 (ObjNum 223 snap 67 seq_num 223) dirty exists 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:683: left oid 25 (ObjNum 223 snap 67 seq_num 223) 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:688: finishing write tid 1 to smithi11075562-17 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:688: finishing write tid 2 to smithi11075562-17 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:688: finishing write tid 3 to smithi11075562-17 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:688: finishing write tid 4 to smithi11075562-17 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:688: finishing write tid 5 to smithi11075562-17 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 304 (ObjNum 224 snap 67 seq_num 224) dirty exists 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:688: left oid 17 (ObjNum 224 snap 67 seq_num 224) 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:684: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:687: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:684: finishing copy_from to smithi11075562-45 2024-09-14T03:39:41.174 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 192 (ObjNum 176 snap 50 seq_num 176) dirty exists 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:687: finishing copy_from to smithi11075562-18 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 195 (ObjNum 209 snap 62 seq_num 209) dirty exists 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:690: finishing write tid 1 to smithi11075562-28 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:690: finishing write tid 2 to smithi11075562-28 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:690: finishing write tid 3 to smithi11075562-28 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:690: finishing write tid 4 to smithi11075562-28 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:690: finishing write tid 5 to smithi11075562-28 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 234 (ObjNum 225 snap 67 seq_num 225) dirty exists 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:690: left oid 28 (ObjNum 225 snap 67 seq_num 225) 2024-09-14T03:39:41.175 INFO:tasks.rados.rados.0.smithi110.stdout:691: expect (ObjNum 186 snap 51 seq_num 186) 2024-09-14T03:39:41.194 INFO:tasks.rados.rados.0.smithi110.stdout:692: finishing write tid 1 to smithi11075562-30 2024-09-14T03:39:41.194 INFO:tasks.rados.rados.0.smithi110.stdout:692: finishing write tid 2 to smithi11075562-30 2024-09-14T03:39:41.195 INFO:tasks.rados.rados.0.smithi110.stdout:692: finishing write tid 3 to smithi11075562-30 2024-09-14T03:39:41.195 INFO:tasks.rados.rados.0.smithi110.stdout:692: finishing write tid 4 to smithi11075562-30 2024-09-14T03:39:41.195 INFO:tasks.rados.rados.0.smithi110.stdout:692: finishing write tid 5 to smithi11075562-30 2024-09-14T03:39:41.195 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 296 (ObjNum 226 snap 67 seq_num 226) dirty exists 2024-09-14T03:39:41.195 INFO:tasks.rados.rados.0.smithi110.stdout:692: left oid 30 (ObjNum 226 snap 67 seq_num 226) 2024-09-14T03:39:41.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:41 smithi195 systemd[1]: Stopping Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:41.779 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:41 smithi195 podman[71847]: 2024-09-14 03:39:41.505215076 +0000 UTC m=+0.134813879 container died 67ea5066f839923cd0a167ee64ef4a838dad38f4406a421590eec7940724c983 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, architecture=x86_64, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-type=git, io.buildah.version=1.32.0, version=8.5, build-date=2022-04-29T11:48:11.055897, name=ubi8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-09-14T03:39:42.136 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:41 smithi195 podman[71847]: 2024-09-14 03:39:41.953660857 +0000 UTC m=+0.583259704 container cleanup 67ea5066f839923cd0a167ee64ef4a838dad38f4406a421590eec7940724c983 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, name=ubi8, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., vcs-type=git, build-date=2022-04-29T11:48:11.055897) 2024-09-14T03:39:42.136 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:41 smithi195 bash[71847]: ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a 2024-09-14T03:39:42.140 INFO:tasks.rados.rados.0.smithi110.stdout:683: done (8 left) 2024-09-14T03:39:42.140 INFO:tasks.rados.rados.0.smithi110.stdout:684: done (7 left) 2024-09-14T03:39:42.140 INFO:tasks.rados.rados.0.smithi110.stdout:685: done (6 left) 2024-09-14T03:39:42.140 INFO:tasks.rados.rados.0.smithi110.stdout:687: done (5 left) 2024-09-14T03:39:42.140 INFO:tasks.rados.rados.0.smithi110.stdout:688: done (4 left) 2024-09-14T03:39:42.140 INFO:tasks.rados.rados.0.smithi110.stdout:690: done (3 left) 2024-09-14T03:39:42.141 INFO:tasks.rados.rados.0.smithi110.stdout:691: done (2 left) 2024-09-14T03:39:42.141 INFO:tasks.rados.rados.0.smithi110.stdout:692: done (1 left) 2024-09-14T03:39:42.141 INFO:tasks.rados.rados.0.smithi110.stdout:693: done (0 left) 2024-09-14T03:39:42.141 INFO:tasks.rados.rados.0.smithi110.stdout:694: delete oid 2 current snap is 68 2024-09-14T03:39:42.142 INFO:tasks.rados.rados.0.smithi110.stdout:694: done (0 left) 2024-09-14T03:39:42.142 INFO:tasks.rados.rados.0.smithi110.stdout:695: delete oid 5 current snap is 68 2024-09-14T03:39:42.146 INFO:tasks.rados.rados.0.smithi110.stdout:695: done (0 left) 2024-09-14T03:39:42.146 INFO:tasks.rados.rados.0.smithi110.stdout:696: read oid 43 snap -1 2024-09-14T03:39:42.147 INFO:tasks.rados.rados.0.smithi110.stdout:696: expect deleted 2024-09-14T03:39:42.147 INFO:tasks.rados.rados.0.smithi110.stdout:697: read oid 28 snap 64 2024-09-14T03:39:42.147 INFO:tasks.rados.rados.0.smithi110.stdout:697: expect (ObjNum 168 snap 48 seq_num 168) 2024-09-14T03:39:42.147 INFO:tasks.rados.rados.0.smithi110.stdout:698: delete oid 17 current snap is 68 2024-09-14T03:39:42.148 INFO:tasks.rados.rados.0.smithi110.stdout:696: done (2 left) 2024-09-14T03:39:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:698: done (1 left) 2024-09-14T03:39:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:699: copy_from oid 45 from oid 13 current snap is 68 2024-09-14T03:39:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:700: rmattr oid 36 current snap is 68 2024-09-14T03:39:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:700: done (2 left) 2024-09-14T03:39:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 68 2024-09-14T03:39:42.149 INFO:tasks.rados.rados.0.smithi110.stdout:701: seq_num 230 ranges {2031616=655360,2686976=491520,3178496=311296} 2024-09-14T03:39:42.163 INFO:tasks.rados.rados.0.smithi110.stdout:701: writing smithi11075562-11 from 2031616 to 2686976 tid 1 2024-09-14T03:39:42.166 INFO:tasks.rados.rados.0.smithi110.stdout:701: writing smithi11075562-11 from 2686976 to 3178496 tid 2 2024-09-14T03:39:42.168 INFO:tasks.rados.rados.0.smithi110.stdout:701: writing smithi11075562-11 from 3178496 to 3489792 tid 3 2024-09-14T03:39:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:702: read oid 19 snap -1 2024-09-14T03:39:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:702: expect deleted 2024-09-14T03:39:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:703: read oid 49 snap 56 2024-09-14T03:39:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:703: expect (ObjNum 193 snap 54 seq_num 193) 2024-09-14T03:39:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:704: rollback oid 44 current snap is 68 2024-09-14T03:39:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 44 to 51 2024-09-14T03:39:42.169 INFO:tasks.rados.rados.0.smithi110.stdout:697: expect (ObjNum 168 snap 48 seq_num 168) 2024-09-14T03:39:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:699: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:39:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:699: finishing copy_from to smithi11075562-45 2024-09-14T03:39:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 278 (ObjNum 140 snap 39 seq_num 140) dirty exists 2024-09-14T03:39:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:701: finishing write tid 1 to smithi11075562-11 2024-09-14T03:39:42.185 INFO:tasks.rados.rados.0.smithi110.stdout:697: done (5 left) 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:699: done (4 left) 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:705: setattr oid 39 current snap is 68 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:701: finishing write tid 2 to smithi11075562-11 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:701: finishing write tid 3 to smithi11075562-11 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:701: finishing write tid 4 to smithi11075562-11 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:701: finishing write tid 5 to smithi11075562-11 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 309 (ObjNum 230 snap 68 seq_num 230) dirty exists 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:701: left oid 11 (ObjNum 230 snap 68 seq_num 230) 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:704: finishing rollback tid 0 to smithi11075562-44 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:704: finishing rollback tid 1 to smithi11075562-44 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:704: finishing rollback tid 2 to smithi11075562-44 2024-09-14T03:39:42.186 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 40 (ObjNum 162 snap 47 seq_num 162) dirty exists 2024-09-14T03:39:42.187 INFO:tasks.rados.rados.0.smithi110.stdout:701: done (4 left) 2024-09-14T03:39:42.187 INFO:tasks.rados.rados.0.smithi110.stdout:702: done (3 left) 2024-09-14T03:39:42.187 INFO:tasks.rados.rados.0.smithi110.stdout:704: done (2 left) 2024-09-14T03:39:42.187 INFO:tasks.rados.rados.0.smithi110.stdout:706: snap_remove snap 57 2024-09-14T03:39:42.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:42 smithi110 ceph-mon[27365]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:39:42.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:42 smithi110 ceph-mon[27365]: pgmap v66: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 584 KiB/s rd, 4.2 MiB/s wr, 23 op/s 2024-09-14T03:39:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:42 smithi110 ceph-mon[35374]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:39:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:42 smithi110 ceph-mon[35374]: pgmap v66: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 584 KiB/s rd, 4.2 MiB/s wr, 23 op/s 2024-09-14T03:39:42.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:42 smithi195 ceph-mon[30695]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:39:42.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:42 smithi195 ceph-mon[30695]: pgmap v66: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 584 KiB/s rd, 4.2 MiB/s wr, 23 op/s 2024-09-14T03:39:42.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:42 smithi195 podman[71860]: 2024-09-14 03:39:42.335402707 +0000 UTC m=+0.827702891 container remove 67ea5066f839923cd0a167ee64ef4a838dad38f4406a421590eec7940724c983 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, distribution-scope=public, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, description=Ceph Grafana Container, name=ubi8, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664) 2024-09-14T03:39:42.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:42 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:39:42.879 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:42 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-14T03:39:42.879 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:42 smithi195 systemd[1]: Stopped Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:42.879 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:42 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service: Consumed 5.081s CPU time. 2024-09-14T03:39:42.879 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:42 smithi195 systemd[1]: Starting Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:42.879 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:42 smithi195 podman[71953]: 2024-09-14 03:39:42.637960583 +0000 UTC m=+0.015079314 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-14T03:39:42.880 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:42 smithi195 podman[71953]: 2024-09-14 03:39:42.765590065 +0000 UTC m=+0.142708827 container create 03236d3bc2106cf24f112656d1d2dea19b5a2b65e8d8e21bf21375c492a44b93 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, name=ubi8, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-14T03:39:43.139 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 podman[71953]: 2024-09-14 03:39:43.010152675 +0000 UTC m=+0.387271400 container init 03236d3bc2106cf24f112656d1d2dea19b5a2b65e8d8e21bf21375c492a44b93 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, name=ubi8, vcs-type=git, maintainer=Paul Cuzner , vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.openshift.tags=base rhel8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.openshift.expose-services=, architecture=x86_64) 2024-09-14T03:39:43.139 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 podman[71953]: 2024-09-14 03:39:43.012991334 +0000 UTC m=+0.390110051 container start 03236d3bc2106cf24f112656d1d2dea19b5a2b65e8d8e21bf21375c492a44b93 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, release=239.1651231664, architecture=x86_64, distribution-scope=public, version=8.5, vendor=Red Hat, Inc., io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-09-14T03:39:43.139 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 bash[71953]: 03236d3bc2106cf24f112656d1d2dea19b5a2b65e8d8e21bf21375c492a44b93 2024-09-14T03:39:43.139 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 systemd[1]: Started Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:43.146 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 356 (ObjNum 208 snap 62 seq_num 208) dirty exists 2024-09-14T03:39:43.146 INFO:tasks.rados.rados.0.smithi110.stdout:703: expect (ObjNum 193 snap 54 seq_num 193) 2024-09-14T03:39:43.295 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:39:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:43 smithi110 ceph-mon[35374]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:39:43.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:43 smithi110 ceph-mon[27365]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:39:43.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-mon[30695]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:39:43.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185286894Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-14T03:39:43.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185519474Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-14T03:39:43.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185544191Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:39:43.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185558479Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:39:43.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185570353Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:39:43.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185585782Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185596574Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185612551Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185625357Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.18563595Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185646059Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.18565549Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185664627Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=settings t=2024-09-14T03:39:43.185680417Z level=info msg="App mode production" 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=sqlstore t=2024-09-14T03:39:43.185738041Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=sqlstore t=2024-09-14T03:39:43.18576166Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=migrator t=2024-09-14T03:39:43.200040649Z level=info msg="Starting DB migrations" 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=migrator t=2024-09-14T03:39:43.204490168Z level=info msg="migrations completed" performed=0 skipped=481 duration=390.907µs 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=sqlstore t=2024-09-14T03:39:43.204862829Z level=info msg="Created default organization" 2024-09-14T03:39:43.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=plugin.loader t=2024-09-14T03:39:43.2613458Z level=info msg="Plugin registered" pluginID=input 2024-09-14T03:39:43.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=plugin.loader t=2024-09-14T03:39:43.289854801Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-14T03:39:43.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=plugin.loader t=2024-09-14T03:39:43.289981529Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-14T03:39:43.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=secrets t=2024-09-14T03:39:43.290081504Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:39:43.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=query_data t=2024-09-14T03:39:43.292172367Z level=info msg="Query Service initialization" 2024-09-14T03:39:43.461 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=live.push_http t=2024-09-14T03:39:43.294669039Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:39:43.535 INFO:tasks.rados.rados.0.smithi110.stdout:703: done (2 left) 2024-09-14T03:39:43.535 INFO:tasks.rados.rados.0.smithi110.stdout:705: done (1 left) 2024-09-14T03:39:43.535 INFO:tasks.rados.rados.0.smithi110.stdout:706: done (0 left) 2024-09-14T03:39:43.535 INFO:tasks.rados.rados.0.smithi110.stdout:707: delete oid 36 current snap is 68 2024-09-14T03:39:43.537 INFO:tasks.rados.rados.0.smithi110.stdout:707: done (0 left) 2024-09-14T03:39:43.537 INFO:tasks.rados.rados.0.smithi110.stdout:708: read oid 42 snap -1 2024-09-14T03:39:43.537 INFO:tasks.rados.rados.0.smithi110.stdout:708: expect (ObjNum 149 snap 44 seq_num 149) 2024-09-14T03:39:43.538 INFO:tasks.rados.rados.0.smithi110.stdout:709: read oid 24 snap -1 2024-09-14T03:39:43.538 INFO:tasks.rados.rados.0.smithi110.stdout:709: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:39:43.538 INFO:tasks.rados.rados.0.smithi110.stdout:710: rmattr oid 45 current snap is 68 2024-09-14T03:39:43.538 INFO:tasks.rados.rados.0.smithi110.stdout:711: snap_remove snap 60 2024-09-14T03:39:43.845 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:39:43.845 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 starting - - - - 2024-09-14T03:39:43.845 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 starting - - - - 2024-09-14T03:39:43.845 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (5m) 38s ago 5m 17.3M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:39:43.845 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (5m) 38s ago 5m 15.8M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:39:43.845 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283 running (64s) 38s ago 9m 499M - 19.3.0-4907-ga2782315 8801d2529e43 2372ccbce64c 2024-09-14T03:39:43.845 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:9283,8765 running (10m) 38s ago 10m 309M - 17.2.7-1629-geb16b97b ab51d1bebd12 a83ea41d9e62 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (10m) 38s ago 11m 54.1M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 c1073676c66c 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (9m) 38s ago 9m 44.5M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 b6415a0de807 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (9m) 38s ago 9m 42.6M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 8e2894672cd8 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 starting - - - - 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (4m) 38s ago 4m 16.4M - 1.5.0 0da6a335fe13 67a66167cf52 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (8m) 38s ago 8m 78.4M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (8m) 38s ago 8m 80.5M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (8m) 38s ago 8m 79.6M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (7m) 38s ago 7m 72.1M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (7m) 38s ago 7m 110M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (7m) 38s ago 7m 85.7M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:39:43.846 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (6m) 38s ago 6m 68.5M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:39:43.847 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (6m) 38s ago 6m 65.8M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:39:43.847 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 starting - - - - 2024-09-14T03:39:44.002 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:43 smithi195 systemd[1]: Stopping Ceph node-exporter.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:44.155 INFO:tasks.rados.rados.0.smithi110.stdout:711: done (3 left) 2024-09-14T03:39:44.155 INFO:tasks.rados.rados.0.smithi110.stdout:712: setattr oid 15 current snap is 68 2024-09-14T03:39:44.155 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 388 (ObjNum 140 snap 39 seq_num 140) dirty exists 2024-09-14T03:39:44.155 INFO:tasks.rados.rados.0.smithi110.stdout:709: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:39:44.184 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:39:44.184 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:39:44.184 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:39:44.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:39:44.184 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:39:44.185 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:39:44.185 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:39:44.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:39:44.185 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:39:44.185 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:39:44.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:39:44.185 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:39:44.186 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:39:44.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:39:44.186 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:39:44.186 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 14, 2024-09-14T03:39:44.186 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:39:44.186 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:39:44.186 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:39:44.203 INFO:tasks.rados.rados.0.smithi110.stdout:708: expect (ObjNum 149 snap 44 seq_num 149) 2024-09-14T03:39:44.236 INFO:tasks.rados.rados.0.smithi110.stdout:708: done (3 left) 2024-09-14T03:39:44.236 INFO:tasks.rados.rados.0.smithi110.stdout:709: done (2 left) 2024-09-14T03:39:44.236 INFO:tasks.rados.rados.0.smithi110.stdout:710: done (1 left) 2024-09-14T03:39:44.236 INFO:tasks.rados.rados.0.smithi110.stdout:713: snap_create 2024-09-14T03:39:44.241 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 320 (ObjNum 167 snap 48 seq_num 167) dirty exists 2024-09-14T03:39:44.274 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:39:44] "GET /metrics HTTP/1.1" 200 36153 "" "Prometheus/2.51.0" 2024-09-14T03:39:44.274 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:44.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:39:44.274 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=infra.usagestats.collector t=2024-09-14T03:39:44.091343051Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:39:44.274 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=provisioning.datasources t=2024-09-14T03:39:44.181172917Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-14T03:39:44.274 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=provisioning.datasources t=2024-09-14T03:39:44.181239819Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-14T03:39:44.274 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=provisioning.datasources t=2024-09-14T03:39:44.192485347Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:39:44.275 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:44 smithi195 podman[72095]: 2024-09-14 03:39:44.001409822 +0000 UTC m=+0.129602292 container died 67a66167cf529ff4c272934e3f565bc9ae53e05553b7649b27abd028f8507dc0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:39:44.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:44.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-mon[30695]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:39:44.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:44.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-mon[30695]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-14T03:39:44.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-mon[30695]: Deploying daemon node-exporter.b on smithi195 2024-09-14T03:39:44.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-mon[30695]: pgmap v69: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 584 KiB/s rd, 4.2 MiB/s wr, 23 op/s 2024-09-14T03:39:44.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-mon[30695]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:44.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:44.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[35374]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:39:44.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:44.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[35374]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-14T03:39:44.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[35374]: Deploying daemon node-exporter.b on smithi195 2024-09-14T03:39:44.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[35374]: pgmap v69: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 584 KiB/s rd, 4.2 MiB/s wr, 23 op/s 2024-09-14T03:39:44.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[35374]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:44.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:44.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[27365]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:39:44.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:44.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[27365]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-14T03:39:44.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[27365]: Deploying daemon node-exporter.b on smithi195 2024-09-14T03:39:44.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[27365]: pgmap v69: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 584 KiB/s rd, 4.2 MiB/s wr, 23 op/s 2024-09-14T03:39:44.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:44 smithi110 ceph-mon[27365]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:44.464 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:39:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:39:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:39:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:39:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [], 2024-09-14T03:39:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "1/20 daemons upgraded", 2024-09-14T03:39:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "", 2024-09-14T03:39:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:39:44.465 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:39:44.636 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:44 smithi195 podman[72095]: 2024-09-14 03:39:44.392267474 +0000 UTC m=+0.520459933 container cleanup 67a66167cf529ff4c272934e3f565bc9ae53e05553b7649b27abd028f8507dc0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:39:44.636 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:44 smithi195 bash[72095]: ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b 2024-09-14T03:39:44.636 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=provisioning.alerting t=2024-09-14T03:39:44.27333545Z level=info msg="starting to provision alerting" 2024-09-14T03:39:44.636 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=provisioning.alerting t=2024-09-14T03:39:44.273376919Z level=info msg="finished to provision alerting" 2024-09-14T03:39:44.637 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=grafanaStorageLogger t=2024-09-14T03:39:44.273845655Z level=info msg="storage starting" 2024-09-14T03:39:44.637 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=ngalert.state.manager t=2024-09-14T03:39:44.273898661Z level=info msg="Warming state cache for startup" 2024-09-14T03:39:44.637 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=http.server t=2024-09-14T03:39:44.275440997Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:39:44.637 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=ngalert.state.manager t=2024-09-14T03:39:44.284836604Z level=info msg="State cache has been initialized" states=0 duration=10.937667ms 2024-09-14T03:39:44.637 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=ticker t=2024-09-14T03:39:44.284889119Z level=info msg=starting first_tick=2024-09-14T03:39:50Z 2024-09-14T03:39:44.637 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:39:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[71967]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:39:44.284910598Z level=info msg="starting MultiOrg Alertmanager" 2024-09-14T03:39:44.959 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:44 smithi195 podman[72108]: 2024-09-14 03:39:44.635305485 +0000 UTC m=+0.637597562 container remove 67a66167cf529ff4c272934e3f565bc9ae53e05553b7649b27abd028f8507dc0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:39:44.959 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:44 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:39:44.960 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:44 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-14T03:39:44.960 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:44 smithi195 systemd[1]: Stopped Ceph node-exporter.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:44.960 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:44 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 2.026s CPU time. 2024-09-14T03:39:45.039 INFO:tasks.rados.rados.0.smithi110.stdout:712: done (1 left) 2024-09-14T03:39:45.040 INFO:tasks.rados.rados.0.smithi110.stdout:713: done (0 left) 2024-09-14T03:39:45.040 INFO:tasks.rados.rados.0.smithi110.stdout:714: rollback oid 47 current snap is 69 2024-09-14T03:39:45.040 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 47 to 63 2024-09-14T03:39:45.040 INFO:tasks.rados.rados.0.smithi110.stdout:715: read oid 10 snap -1 2024-09-14T03:39:45.040 INFO:tasks.rados.rados.0.smithi110.stdout:715: expect deleted 2024-09-14T03:39:45.040 INFO:tasks.rados.rados.0.smithi110.stdout:716: read oid 42 snap 51 2024-09-14T03:39:45.040 INFO:tasks.rados.rados.0.smithi110.stdout:716: expect (ObjNum 180 snap 51 seq_num 180) 2024-09-14T03:39:45.040 INFO:tasks.rados.rados.0.smithi110.stdout:717: snap_remove snap 21 2024-09-14T03:39:45.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[27365]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:45.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[27365]: from='client.15108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:45.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[27365]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:39:45.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3893289199' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:39:45.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[27365]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:39:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[35374]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[35374]: from='client.15108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[35374]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:39:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3893289199' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:39:45.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:45 smithi110 ceph-mon[35374]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:39:45.459 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:45 smithi195 systemd[1]: Starting Ceph node-exporter.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:45.459 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:45 smithi195 bash[72238]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-14T03:39:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:45 smithi195 ceph-mon[30695]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:45 smithi195 ceph-mon[30695]: from='client.15108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:45.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:45 smithi195 ceph-mon[30695]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:39:45.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:45 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3893289199' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:39:45.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:45 smithi195 ceph-mon[30695]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:39:45.958 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:45 smithi195 bash[72238]: Getting image source signatures 2024-09-14T03:39:45.959 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:45 smithi195 bash[72238]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-14T03:39:45.959 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:45 smithi195 bash[72238]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-14T03:39:45.959 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:45 smithi195 bash[72238]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-14T03:39:45.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:45.739Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:39:45.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:39:46.041 INFO:tasks.rados.rados.0.smithi110.stdout:716: expect (ObjNum 180 snap 51 seq_num 180) 2024-09-14T03:39:46.105 INFO:tasks.rados.rados.0.smithi110.stdout:714: finishing rollback tid 0 to smithi11075562-47 2024-09-14T03:39:46.105 INFO:tasks.rados.rados.0.smithi110.stdout:714: finishing rollback tid 1 to smithi11075562-47 2024-09-14T03:39:46.105 INFO:tasks.rados.rados.0.smithi110.stdout:714: finishing rollback tid 2 to smithi11075562-47 2024-09-14T03:39:46.105 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 400 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:39:46.105 INFO:tasks.rados.rados.0.smithi110.stdout:714: done (3 left) 2024-09-14T03:39:46.105 INFO:tasks.rados.rados.0.smithi110.stdout:715: done (2 left) 2024-09-14T03:39:46.106 INFO:tasks.rados.rados.0.smithi110.stdout:716: done (1 left) 2024-09-14T03:39:46.106 INFO:tasks.rados.rados.0.smithi110.stdout:717: done (0 left) 2024-09-14T03:39:46.106 INFO:tasks.rados.rados.0.smithi110.stdout:718: rmattr oid 31 current snap is 69 2024-09-14T03:39:46.106 INFO:tasks.rados.rados.0.smithi110.stdout:718: done (0 left) 2024-09-14T03:39:46.106 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 69 2024-09-14T03:39:46.106 INFO:tasks.rados.rados.0.smithi110.stdout:719: seq_num 232 ranges {778240=688128,1466368=663552,2129920=139264} 2024-09-14T03:39:46.112 INFO:tasks.rados.rados.0.smithi110.stdout:719: writing smithi11075562-29 from 778240 to 1466368 tid 1 2024-09-14T03:39:46.116 INFO:tasks.rados.rados.0.smithi110.stdout:719: writing smithi11075562-29 from 1466368 to 2129920 tid 2 2024-09-14T03:39:46.117 INFO:tasks.rados.rados.0.smithi110.stdout:719: writing smithi11075562-29 from 2129920 to 2269184 tid 3 2024-09-14T03:39:46.117 INFO:tasks.rados.rados.0.smithi110.stdout:720: read oid 40 snap -1 2024-09-14T03:39:46.117 INFO:tasks.rados.rados.0.smithi110.stdout:720: expect deleted 2024-09-14T03:39:46.118 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 69 2024-09-14T03:39:46.118 INFO:tasks.rados.rados.0.smithi110.stdout:719: finishing write tid 1 to smithi11075562-29 2024-09-14T03:39:46.118 INFO:tasks.rados.rados.0.smithi110.stdout:721: seq_num 233 ranges {0=712704,712704=548864,1261568=253952} 2024-09-14T03:39:46.121 INFO:tasks.rados.rados.0.smithi110.stdout:721: writing smithi11075562-12 from 0 to 712704 tid 1 2024-09-14T03:39:46.124 INFO:tasks.rados.rados.0.smithi110.stdout:721: writing smithi11075562-12 from 712704 to 1261568 tid 2 2024-09-14T03:39:46.126 INFO:tasks.rados.rados.0.smithi110.stdout:721: writing smithi11075562-12 from 1261568 to 1515520 tid 3 2024-09-14T03:39:46.126 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 69 2024-09-14T03:39:46.126 INFO:tasks.rados.rados.0.smithi110.stdout:722: seq_num 234 ranges {0=729088,729088=434176,1163264=368640} 2024-09-14T03:39:46.130 INFO:tasks.rados.rados.0.smithi110.stdout:722: writing smithi11075562-16 from 0 to 729088 tid 1 2024-09-14T03:39:46.133 INFO:tasks.rados.rados.0.smithi110.stdout:722: writing smithi11075562-16 from 729088 to 1163264 tid 2 2024-09-14T03:39:46.135 INFO:tasks.rados.rados.0.smithi110.stdout:722: writing smithi11075562-16 from 1163264 to 1531904 tid 3 2024-09-14T03:39:46.135 INFO:tasks.rados.rados.0.smithi110.stdout:723: read oid 36 snap 68 2024-09-14T03:39:46.135 INFO:tasks.rados.rados.0.smithi110.stdout:723: expect deleted 2024-09-14T03:39:46.135 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 69 2024-09-14T03:39:46.135 INFO:tasks.rados.rados.0.smithi110.stdout:724: seq_num 235 ranges {0=745472,745472=712704,1458176=90112} 2024-09-14T03:39:46.139 INFO:tasks.rados.rados.0.smithi110.stdout:724: writing smithi11075562-19 from 0 to 745472 tid 1 2024-09-14T03:39:46.143 INFO:tasks.rados.rados.0.smithi110.stdout:724: writing smithi11075562-19 from 745472 to 1458176 tid 2 2024-09-14T03:39:46.144 INFO:tasks.rados.rados.0.smithi110.stdout:724: writing smithi11075562-19 from 1458176 to 1548288 tid 3 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:725: rmattr oid 21 current snap is 69 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:725: done (6 left) 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:726: rmattr oid 33 current snap is 69 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:726: done (6 left) 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:727: rmattr oid 42 current snap is 69 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:727: done (6 left) 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 69 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:719: finishing write tid 2 to smithi11075562-29 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:719: finishing write tid 3 to smithi11075562-29 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:719: finishing write tid 4 to smithi11075562-29 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:719: finishing write tid 5 to smithi11075562-29 2024-09-14T03:39:46.145 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 167 (ObjNum 232 snap 69 seq_num 232) dirty exists 2024-09-14T03:39:46.146 INFO:tasks.rados.rados.0.smithi110.stdout:719: left oid 29 (ObjNum 232 snap 69 seq_num 232) 2024-09-14T03:39:46.146 INFO:tasks.rados.rados.0.smithi110.stdout:721: finishing write tid 1 to smithi11075562-12 2024-09-14T03:39:46.146 INFO:tasks.rados.rados.0.smithi110.stdout:728: seq_num 236 ranges {720896=761856,1482752=647168,2129920=155648} 2024-09-14T03:39:46.152 INFO:tasks.rados.rados.0.smithi110.stdout:728: writing smithi11075562-35 from 720896 to 1482752 tid 1 2024-09-14T03:39:46.155 INFO:tasks.rados.rados.0.smithi110.stdout:728: writing smithi11075562-35 from 1482752 to 2129920 tid 2 2024-09-14T03:39:46.157 INFO:tasks.rados.rados.0.smithi110.stdout:728: writing smithi11075562-35 from 2129920 to 2285568 tid 3 2024-09-14T03:39:46.157 INFO:tasks.rados.rados.0.smithi110.stdout:719: done (6 left) 2024-09-14T03:39:46.157 INFO:tasks.rados.rados.0.smithi110.stdout:720: done (5 left) 2024-09-14T03:39:46.157 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 69 2024-09-14T03:39:46.157 INFO:tasks.rados.rados.0.smithi110.stdout:721: finishing write tid 2 to smithi11075562-12 2024-09-14T03:39:46.158 INFO:tasks.rados.rados.0.smithi110.stdout:721: finishing write tid 3 to smithi11075562-12 2024-09-14T03:39:46.158 INFO:tasks.rados.rados.0.smithi110.stdout:721: finishing write tid 4 to smithi11075562-12 2024-09-14T03:39:46.158 INFO:tasks.rados.rados.0.smithi110.stdout:721: finishing write tid 5 to smithi11075562-12 2024-09-14T03:39:46.158 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 414 (ObjNum 233 snap 69 seq_num 233) dirty exists 2024-09-14T03:39:46.158 INFO:tasks.rados.rados.0.smithi110.stdout:721: left oid 12 (ObjNum 233 snap 69 seq_num 233) 2024-09-14T03:39:46.158 INFO:tasks.rados.rados.0.smithi110.stdout:722: finishing write tid 1 to smithi11075562-16 2024-09-14T03:39:46.158 INFO:tasks.rados.rados.0.smithi110.stdout:722: finishing write tid 2 to smithi11075562-16 2024-09-14T03:39:46.158 INFO:tasks.rados.rados.0.smithi110.stdout:729: seq_num 237 ranges {3915776=778240,4694016=532480,5226496=270336} 2024-09-14T03:39:46.182 INFO:tasks.rados.rados.0.smithi110.stdout:729: writing smithi11075562-13 from 3915776 to 4694016 tid 1 2024-09-14T03:39:46.185 INFO:tasks.rados.rados.0.smithi110.stdout:729: writing smithi11075562-13 from 4694016 to 5226496 tid 2 2024-09-14T03:39:46.187 INFO:tasks.rados.rados.0.smithi110.stdout:729: writing smithi11075562-13 from 5226496 to 5496832 tid 3 2024-09-14T03:39:46.187 INFO:tasks.rados.rados.0.smithi110.stdout:721: done (5 left) 2024-09-14T03:39:46.187 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 69 2024-09-14T03:39:46.187 INFO:tasks.rados.rados.0.smithi110.stdout:730: seq_num 238 ranges {1277952=794624,2072576=417792,2490368=385024} 2024-09-14T03:39:46.198 INFO:tasks.rados.rados.0.smithi110.stdout:730: writing smithi11075562-41 from 1277952 to 2072576 tid 1 2024-09-14T03:39:46.200 INFO:tasks.rados.rados.0.smithi110.stdout:730: writing smithi11075562-41 from 2072576 to 2490368 tid 2 2024-09-14T03:39:46.203 INFO:tasks.rados.rados.0.smithi110.stdout:730: writing smithi11075562-41 from 2490368 to 2875392 tid 3 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:731: read oid 6 snap -1 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:731: expect (ObjNum 176 snap 50 seq_num 176) 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:732: read oid 39 snap -1 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:732: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:733: delete oid 3 current snap is 69 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:722: finishing write tid 3 to smithi11075562-16 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:722: finishing write tid 4 to smithi11075562-16 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:722: finishing write tid 5 to smithi11075562-16 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 319 (ObjNum 234 snap 69 seq_num 234) dirty exists 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:722: left oid 16 (ObjNum 234 snap 69 seq_num 234) 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:724: finishing write tid 1 to smithi11075562-19 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:724: finishing write tid 2 to smithi11075562-19 2024-09-14T03:39:46.204 INFO:tasks.rados.rados.0.smithi110.stdout:724: finishing write tid 3 to smithi11075562-19 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:724: finishing write tid 4 to smithi11075562-19 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:724: finishing write tid 5 to smithi11075562-19 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 323 (ObjNum 235 snap 69 seq_num 235) dirty exists 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:724: left oid 19 (ObjNum 235 snap 69 seq_num 235) 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:728: finishing write tid 1 to smithi11075562-35 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:728: finishing write tid 2 to smithi11075562-35 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:728: finishing write tid 3 to smithi11075562-35 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:728: finishing write tid 4 to smithi11075562-35 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:728: finishing write tid 5 to smithi11075562-35 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 365 (ObjNum 236 snap 69 seq_num 236) dirty exists 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:728: left oid 35 (ObjNum 236 snap 69 seq_num 236) 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:729: finishing write tid 1 to smithi11075562-13 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:729: finishing write tid 2 to smithi11075562-13 2024-09-14T03:39:46.205 INFO:tasks.rados.rados.0.smithi110.stdout:729: finishing write tid 3 to smithi11075562-13 2024-09-14T03:39:46.206 INFO:tasks.rados.rados.0.smithi110.stdout:729: finishing write tid 4 to smithi11075562-13 2024-09-14T03:39:46.206 INFO:tasks.rados.rados.0.smithi110.stdout:729: finishing write tid 5 to smithi11075562-13 2024-09-14T03:39:46.206 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 370 (ObjNum 237 snap 69 seq_num 237) dirty exists 2024-09-14T03:39:46.206 INFO:tasks.rados.rados.0.smithi110.stdout:729: left oid 13 (ObjNum 237 snap 69 seq_num 237) 2024-09-14T03:39:46.206 INFO:tasks.rados.rados.0.smithi110.stdout:730: finishing write tid 1 to smithi11075562-41 2024-09-14T03:39:46.206 INFO:tasks.rados.rados.0.smithi110.stdout:730: finishing write tid 2 to smithi11075562-41 2024-09-14T03:39:46.207 INFO:tasks.rados.rados.0.smithi110.stdout:722: done (8 left) 2024-09-14T03:39:46.207 INFO:tasks.rados.rados.0.smithi110.stdout:723: done (7 left) 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:724: done (6 left) 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:728: done (5 left) 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:729: done (4 left) 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:733: done (3 left) 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:734: delete oid 12 current snap is 69 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:730: finishing write tid 3 to smithi11075562-41 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:730: finishing write tid 4 to smithi11075562-41 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:734: done (3 left) 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:735: rmattr oid 15 current snap is 69 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:736: read oid 18 snap -1 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:736: expect (ObjNum 209 snap 62 seq_num 209) 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:737: copy_from oid 45 from oid 29 current snap is 69 2024-09-14T03:39:46.208 INFO:tasks.rados.rados.0.smithi110.stdout:738: read oid 20 snap -1 2024-09-14T03:39:46.209 INFO:tasks.rados.rados.0.smithi110.stdout:738: expect deleted 2024-09-14T03:39:46.209 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 69 2024-09-14T03:39:46.209 INFO:tasks.rados.rados.0.smithi110.stdout:739: seq_num 241 ranges {3538944=442368,3981312=466944,4448256=688128,5136384=49152} 2024-09-14T03:39:46.232 INFO:tasks.rados.rados.0.smithi110.stdout:739: writing smithi11075562-26 from 3538944 to 3981312 tid 1 2024-09-14T03:39:46.235 INFO:tasks.rados.rados.0.smithi110.stdout:739: writing smithi11075562-26 from 3981312 to 4448256 tid 2 2024-09-14T03:39:46.239 INFO:tasks.rados.rados.0.smithi110.stdout:739: writing smithi11075562-26 from 4448256 to 5136384 tid 3 2024-09-14T03:39:46.240 INFO:tasks.rados.rados.0.smithi110.stdout:739: writing smithi11075562-26 from 5136384 to 5185536 tid 4 2024-09-14T03:39:46.240 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 69 2024-09-14T03:39:46.240 INFO:tasks.rados.rados.0.smithi110.stdout:740: seq_num 242 ranges {8970240=458752,9428992=753664,10182656=450560} 2024-09-14T03:39:46.291 INFO:tasks.rados.rados.0.smithi110.stdout:740: writing smithi11075562-38 from 8970240 to 9428992 tid 1 2024-09-14T03:39:46.294 INFO:tasks.rados.rados.0.smithi110.stdout:740: writing smithi11075562-38 from 9428992 to 10182656 tid 2 2024-09-14T03:39:46.297 INFO:tasks.rados.rados.0.smithi110.stdout:740: writing smithi11075562-38 from 10182656 to 10633216 tid 3 2024-09-14T03:39:46.297 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 69 2024-09-14T03:39:46.297 INFO:tasks.rados.rados.0.smithi110.stdout:741: seq_num 243 ranges {794624=475136,1269760=638976,1908736=565248} 2024-09-14T03:39:46.304 INFO:tasks.rados.rados.0.smithi110.stdout:741: writing smithi11075562-42 from 794624 to 1269760 tid 1 2024-09-14T03:39:46.307 INFO:tasks.rados.rados.0.smithi110.stdout:741: writing smithi11075562-42 from 1269760 to 1908736 tid 2 2024-09-14T03:39:46.311 INFO:tasks.rados.rados.0.smithi110.stdout:741: writing smithi11075562-42 from 1908736 to 2473984 tid 3 2024-09-14T03:39:46.311 INFO:tasks.rados.rados.0.smithi110.stdout:742: copy_from oid 10 from oid 28 current snap is 69 2024-09-14T03:39:46.311 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 322 (ObjNum 167 snap 48 seq_num 167) dirty exists 2024-09-14T03:39:46.311 INFO:tasks.rados.rados.0.smithi110.stdout:730: finishing write tid 5 to smithi11075562-41 2024-09-14T03:39:46.311 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 381 (ObjNum 238 snap 69 seq_num 238) dirty exists 2024-09-14T03:39:46.311 INFO:tasks.rados.rados.0.smithi110.stdout:730: left oid 41 (ObjNum 238 snap 69 seq_num 238) 2024-09-14T03:39:46.312 INFO:tasks.rados.rados.0.smithi110.stdout:732: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:39:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:46 smithi110 ceph-mon[35374]: from='client.15120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:46 smithi110 ceph-mon[35374]: pgmap v72: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 25 op/s 2024-09-14T03:39:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:46 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:46.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:46 smithi110 ceph-mon[35374]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:39:46.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:46 smithi110 ceph-mon[27365]: from='client.15120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:46.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:46 smithi110 ceph-mon[27365]: pgmap v72: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 25 op/s 2024-09-14T03:39:46.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:46 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:46.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:46 smithi110 ceph-mon[27365]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:39:46.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:46 smithi195 ceph-mon[30695]: from='client.15120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:46.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:46 smithi195 ceph-mon[30695]: pgmap v72: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 25 op/s 2024-09-14T03:39:46.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:46 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:39:46.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:46 smithi195 ceph-mon[30695]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:39:46.525 INFO:tasks.rados.rados.0.smithi110.stdout:731: expect (ObjNum 176 snap 50 seq_num 176) 2024-09-14T03:39:46.710 INFO:tasks.rados.rados.0.smithi110.stdout:737: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:39:46.711 INFO:tasks.rados.rados.0.smithi110.stdout:737: finishing copy_from to smithi11075562-45 2024-09-14T03:39:46.711 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 167 (ObjNum 232 snap 69 seq_num 232) dirty exists 2024-09-14T03:39:46.711 INFO:tasks.rados.rados.0.smithi110.stdout:739: finishing write tid 1 to smithi11075562-26 2024-09-14T03:39:46.711 INFO:tasks.rados.rados.0.smithi110.stdout:739: finishing write tid 2 to smithi11075562-26 2024-09-14T03:39:46.711 INFO:tasks.rados.rados.0.smithi110.stdout:736: expect (ObjNum 209 snap 62 seq_num 209) 2024-09-14T03:39:46.831 INFO:tasks.rados.rados.0.smithi110.stdout:739: finishing write tid 3 to smithi11075562-26 2024-09-14T03:39:46.831 INFO:tasks.rados.rados.0.smithi110.stdout:739: finishing write tid 4 to smithi11075562-26 2024-09-14T03:39:46.831 INFO:tasks.rados.rados.0.smithi110.stdout:739: finishing write tid 5 to smithi11075562-26 2024-09-14T03:39:46.831 INFO:tasks.rados.rados.0.smithi110.stdout:739: finishing write tid 6 to smithi11075562-26 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 133 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:739: left oid 26 (ObjNum 241 snap 69 seq_num 241) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:730: done (10 left) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:731: done (9 left) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:732: done (8 left) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:735: done (7 left) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:736: done (6 left) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:737: done (5 left) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:738: done (4 left) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:739: done (3 left) 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 69 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:740: finishing write tid 1 to smithi11075562-38 2024-09-14T03:39:46.832 INFO:tasks.rados.rados.0.smithi110.stdout:743: seq_num 244 ranges {4096000=491520,4587520=573440,5160960=458752,5619712=172032} 2024-09-14T03:39:46.854 INFO:tasks.rados.rados.0.smithi110.stdout:743: writing smithi11075562-6 from 4096000 to 4587520 tid 1 2024-09-14T03:39:46.857 INFO:tasks.rados.rados.0.smithi110.stdout:743: writing smithi11075562-6 from 4587520 to 5160960 tid 2 2024-09-14T03:39:46.860 INFO:tasks.rados.rados.0.smithi110.stdout:743: writing smithi11075562-6 from 5160960 to 5619712 tid 3 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:743: writing smithi11075562-6 from 5619712 to 5791744 tid 4 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:744: delete oid 30 current snap is 69 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:740: finishing write tid 2 to smithi11075562-38 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:740: finishing write tid 3 to smithi11075562-38 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:740: finishing write tid 4 to smithi11075562-38 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:740: finishing write tid 5 to smithi11075562-38 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 172 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:740: left oid 38 (ObjNum 242 snap 69 seq_num 242) 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:741: finishing write tid 1 to smithi11075562-42 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:741: finishing write tid 2 to smithi11075562-42 2024-09-14T03:39:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:741: finishing write tid 3 to smithi11075562-42 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:741: finishing write tid 4 to smithi11075562-42 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:741: finishing write tid 5 to smithi11075562-42 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 375 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:741: left oid 42 (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:742: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:742: finishing copy_from to smithi11075562-10 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 234 (ObjNum 225 snap 67 seq_num 225) dirty exists 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:743: finishing write tid 1 to smithi11075562-6 2024-09-14T03:39:46.863 INFO:tasks.rados.rados.0.smithi110.stdout:743: finishing write tid 2 to smithi11075562-6 2024-09-14T03:39:46.864 INFO:tasks.rados.rados.0.smithi110.stdout:743: finishing write tid 3 to smithi11075562-6 2024-09-14T03:39:46.864 INFO:tasks.rados.rados.0.smithi110.stdout:743: finishing write tid 4 to smithi11075562-6 2024-09-14T03:39:46.865 INFO:tasks.rados.rados.0.smithi110.stdout:740: done (4 left) 2024-09-14T03:39:46.865 INFO:tasks.rados.rados.0.smithi110.stdout:741: done (3 left) 2024-09-14T03:39:46.865 INFO:tasks.rados.rados.0.smithi110.stdout:742: done (2 left) 2024-09-14T03:39:46.865 INFO:tasks.rados.rados.0.smithi110.stdout:744: done (1 left) 2024-09-14T03:39:46.865 INFO:tasks.rados.rados.0.smithi110.stdout:745: snap_remove snap 68 2024-09-14T03:39:47.165 INFO:tasks.rados.rados.0.smithi110.stdout:745: done (1 left) 2024-09-14T03:39:47.165 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 69 2024-09-14T03:39:47.165 INFO:tasks.rados.rados.0.smithi110.stdout:746: seq_num 246 ranges {2285568=524288,2809856=745472,3555328=425984,3981312=32768} 2024-09-14T03:39:47.180 INFO:tasks.rados.rados.0.smithi110.stdout:746: writing smithi11075562-35 from 2285568 to 2809856 tid 1 2024-09-14T03:39:47.184 INFO:tasks.rados.rados.0.smithi110.stdout:746: writing smithi11075562-35 from 2809856 to 3555328 tid 2 2024-09-14T03:39:47.187 INFO:tasks.rados.rados.0.smithi110.stdout:746: writing smithi11075562-35 from 3555328 to 3981312 tid 3 2024-09-14T03:39:47.187 INFO:tasks.rados.rados.0.smithi110.stdout:746: writing smithi11075562-35 from 3981312 to 4014080 tid 4 2024-09-14T03:39:47.187 INFO:tasks.rados.rados.0.smithi110.stdout:747: read oid 17 snap -1 2024-09-14T03:39:47.187 INFO:tasks.rados.rados.0.smithi110.stdout:747: expect deleted 2024-09-14T03:39:47.187 INFO:tasks.rados.rados.0.smithi110.stdout:748: snap_create 2024-09-14T03:39:47.187 INFO:tasks.rados.rados.0.smithi110.stdout:743: finishing write tid 5 to smithi11075562-6 2024-09-14T03:39:47.187 INFO:tasks.rados.rados.0.smithi110.stdout:743: finishing write tid 6 to smithi11075562-6 2024-09-14T03:39:47.188 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 329 (ObjNum 244 snap 69 seq_num 244) dirty exists 2024-09-14T03:39:47.188 INFO:tasks.rados.rados.0.smithi110.stdout:743: left oid 6 (ObjNum 244 snap 69 seq_num 244) 2024-09-14T03:39:47.188 INFO:tasks.rados.rados.0.smithi110.stdout:746: finishing write tid 1 to smithi11075562-35 2024-09-14T03:39:47.188 INFO:tasks.rados.rados.0.smithi110.stdout:746: finishing write tid 2 to smithi11075562-35 2024-09-14T03:39:47.189 INFO:tasks.rados.rados.0.smithi110.stdout:746: finishing write tid 3 to smithi11075562-35 2024-09-14T03:39:47.190 INFO:tasks.rados.rados.0.smithi110.stdout:746: finishing write tid 4 to smithi11075562-35 2024-09-14T03:39:47.190 INFO:tasks.rados.rados.0.smithi110.stdout:746: finishing write tid 5 to smithi11075562-35 2024-09-14T03:39:47.191 INFO:tasks.rados.rados.0.smithi110.stdout:746: finishing write tid 6 to smithi11075562-35 2024-09-14T03:39:47.191 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 386 (ObjNum 246 snap 69 seq_num 246) dirty exists 2024-09-14T03:39:47.191 INFO:tasks.rados.rados.0.smithi110.stdout:746: left oid 35 (ObjNum 246 snap 69 seq_num 246) 2024-09-14T03:39:47.208 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:46 smithi195 bash[72238]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-14T03:39:47.208 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 bash[72238]: Writing manifest to image destination 2024-09-14T03:39:47.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:39:47.797 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 podman[72238]: 2024-09-14 03:39:47.447914724 +0000 UTC m=+2.329105479 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-14T03:39:47.797 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 podman[72238]: 2024-09-14 03:39:47.579968003 +0000 UTC m=+2.461158741 container create e217e492de5d7738c7f7962a51fed0d5d79d17c99436cc4f3a142ffdb6eae0db (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:39:47.797 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 podman[72238]: 2024-09-14 03:39:47.790432561 +0000 UTC m=+2.671623300 container init e217e492de5d7738c7f7962a51fed0d5d79d17c99436cc4f3a142ffdb6eae0db (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:39:47.797 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 podman[72238]: 2024-09-14 03:39:47.793433501 +0000 UTC m=+2.674624241 container start e217e492de5d7738c7f7962a51fed0d5d79d17c99436cc4f3a142ffdb6eae0db (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:39:48.165 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.796Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.796Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.796Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:39:48.166 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:39:48.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.797Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:39:48.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:39:48.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b[72314]: ts=2024-09-14T03:39:47.798Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:39:48.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 bash[72238]: e217e492de5d7738c7f7962a51fed0d5d79d17c99436cc4f3a142ffdb6eae0db 2024-09-14T03:39:48.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:39:47 smithi195 systemd[1]: Started Ceph node-exporter.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:48.172 INFO:tasks.rados.rados.0.smithi110.stdout:743: done (3 left) 2024-09-14T03:39:48.172 INFO:tasks.rados.rados.0.smithi110.stdout:746: done (2 left) 2024-09-14T03:39:48.172 INFO:tasks.rados.rados.0.smithi110.stdout:747: done (1 left) 2024-09-14T03:39:48.172 INFO:tasks.rados.rados.0.smithi110.stdout:748: done (0 left) 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:749: setattr oid 33 current snap is 70 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:750: read oid 6 snap -1 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:750: expect (ObjNum 244 snap 69 seq_num 244) 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:751: delete oid 27 current snap is 70 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:751: done (2 left) 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:752: rollback oid 23 current snap is 70 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 23 to 64 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:753: read oid 39 snap -1 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:753: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:754: snap_create 2024-09-14T03:39:48.173 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 252 (ObjNum 203 snap 58 seq_num 203) dirty exists 2024-09-14T03:39:48.175 INFO:tasks.rados.rados.0.smithi110.stdout:752: finishing rollback tid 0 to smithi11075562-23 2024-09-14T03:39:48.176 INFO:tasks.rados.rados.0.smithi110.stdout:752: finishing rollback tid 1 to smithi11075562-23 2024-09-14T03:39:48.176 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 236 (ObjNum 81 snap 0 seq_num 1783655281) dirty dne 2024-09-14T03:39:48.183 INFO:tasks.rados.rados.0.smithi110.stdout:753: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:39:48.406 INFO:tasks.rados.rados.0.smithi110.stdout:750: expect (ObjNum 244 snap 69 seq_num 244) 2024-09-14T03:39:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[27365]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:39:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[27365]: pgmap v75: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 25 op/s 2024-09-14T03:39:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:48.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:39:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:39:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:39:48.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[35374]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:39:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[35374]: pgmap v75: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 25 op/s 2024-09-14T03:39:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:48.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:39:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:39:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:39:48.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:48 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:48.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:48 smithi195 ceph-mon[30695]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:39:48.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:48 smithi195 ceph-mon[30695]: pgmap v75: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 25 op/s 2024-09-14T03:39:48.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:48 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:48.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:48 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:48.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:48 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:39:48.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:48 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:39:48.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:48 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:39:48.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:48 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:49.171 INFO:tasks.rados.rados.0.smithi110.stdout:749: done (4 left) 2024-09-14T03:39:49.171 INFO:tasks.rados.rados.0.smithi110.stdout:750: done (3 left) 2024-09-14T03:39:49.171 INFO:tasks.rados.rados.0.smithi110.stdout:752: done (2 left) 2024-09-14T03:39:49.171 INFO:tasks.rados.rados.0.smithi110.stdout:753: done (1 left) 2024-09-14T03:39:49.172 INFO:tasks.rados.rados.0.smithi110.stdout:754: done (0 left) 2024-09-14T03:39:49.172 INFO:tasks.rados.rados.0.smithi110.stdout:755: setattr oid 26 current snap is 71 2024-09-14T03:39:49.172 INFO:tasks.rados.rados.0.smithi110.stdout:756: snap_remove snap 62 2024-09-14T03:39:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:39:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:39:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:39:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[35374]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:39:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[35374]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:39:49.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[35374]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:39:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:39:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:39:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[27365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:39:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[27365]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:39:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[27365]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:39:49.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:49 smithi110 ceph-mon[27365]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:39:49.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:49 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:39:49.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:49 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:39:49.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:49 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:39:49.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:49 smithi195 ceph-mon[30695]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:39:49.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:49 smithi195 ceph-mon[30695]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:39:49.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:49 smithi195 ceph-mon[30695]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:39:50.059 INFO:tasks.rados.rados.0.smithi110.stdout:756: done (1 left) 2024-09-14T03:39:50.059 INFO:tasks.rados.rados.0.smithi110.stdout:757: delete oid 10 current snap is 71 2024-09-14T03:39:50.059 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 134 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:39:50.066 INFO:tasks.rados.rados.0.smithi110.stdout:755: done (1 left) 2024-09-14T03:39:50.067 INFO:tasks.rados.rados.0.smithi110.stdout:757: done (0 left) 2024-09-14T03:39:50.067 INFO:tasks.rados.rados.0.smithi110.stdout:758: snap_create 2024-09-14T03:39:50.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[35374]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:39:50.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[35374]: pgmap v78: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:50.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:50.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:39:50.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:39:50.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:39:50.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[35374]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[27365]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[27365]: pgmap v78: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:50.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:50 smithi110 ceph-mon[27365]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:39:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:50 smithi195 ceph-mon[30695]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:39:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:50 smithi195 ceph-mon[30695]: pgmap v78: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:39:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:50 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:50 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:39:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:50 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:39:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:50 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:39:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:50 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:50.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:50 smithi195 ceph-mon[30695]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:39:51.042 INFO:tasks.rados.rados.0.smithi110.stdout:758: done (0 left) 2024-09-14T03:39:51.042 INFO:tasks.rados.rados.0.smithi110.stdout:759: setattr oid 43 current snap is 72 2024-09-14T03:39:51.043 INFO:tasks.rados.rados.0.smithi110.stdout:760: snap_remove snap 39 2024-09-14T03:39:51.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:51 smithi110 systemd[1]: Stopping Ceph mgr.y for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:51.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:51 smithi110 ceph-mon[27365]: Upgrade: Updating mgr.y 2024-09-14T03:39:51.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:51 smithi110 ceph-mon[27365]: Deploying daemon mgr.y on smithi110 2024-09-14T03:39:51.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:51 smithi110 ceph-mon[27365]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:39:51.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:51 smithi110 ceph-mon[35374]: Upgrade: Updating mgr.y 2024-09-14T03:39:51.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:51 smithi110 ceph-mon[35374]: Deploying daemon mgr.y on smithi110 2024-09-14T03:39:51.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:51 smithi110 ceph-mon[35374]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:39:51.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:51 smithi195 ceph-mon[30695]: Upgrade: Updating mgr.y 2024-09-14T03:39:51.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:51 smithi195 ceph-mon[30695]: Deploying daemon mgr.y on smithi110 2024-09-14T03:39:51.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:51 smithi195 ceph-mon[30695]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:39:51.760 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:51 smithi110 podman[78518]: 2024-09-14 03:39:51.471328347 +0000 UTC m=+0.128174694 container died a83ea41d9e621f02b7cf03153601bd44f0e268a8794429303de6d8032a6045fe (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-14T03:39:52.043 INFO:tasks.rados.rados.0.smithi110.stdout:760: done (1 left) 2024-09-14T03:39:52.044 INFO:tasks.rados.rados.0.smithi110.stdout:761: read oid 21 snap -1 2024-09-14T03:39:52.044 INFO:tasks.rados.rados.0.smithi110.stdout:761: expect deleted 2024-09-14T03:39:52.044 INFO:tasks.rados.rados.0.smithi110.stdout:762: snap_remove snap 20 2024-09-14T03:39:52.179 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:51 smithi110 podman[78518]: 2024-09-14 03:39:51.934042363 +0000 UTC m=+0.590888750 container cleanup a83ea41d9e621f02b7cf03153601bd44f0e268a8794429303de6d8032a6045fe (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=quincy-eb16b97, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True) 2024-09-14T03:39:52.179 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:51 smithi110 bash[78518]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y 2024-09-14T03:39:52.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:52 smithi110 ceph-mon[27365]: pgmap v81: 121 pgs: 121 active+clean; 206 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.9 MiB/s wr, 36 op/s 2024-09-14T03:39:52.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:52 smithi110 ceph-mon[27365]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:39:52.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:52 smithi110 podman[78531]: 2024-09-14 03:39:52.294784476 +0000 UTC m=+0.817104023 container remove a83ea41d9e621f02b7cf03153601bd44f0e268a8794429303de6d8032a6045fe (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:39:52.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:52 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:39:52.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:52 smithi110 ceph-mon[35374]: pgmap v81: 121 pgs: 121 active+clean; 206 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.9 MiB/s wr, 36 op/s 2024-09-14T03:39:52.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:52 smithi110 ceph-mon[35374]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:39:52.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:52 smithi195 ceph-mon[30695]: pgmap v81: 121 pgs: 121 active+clean; 206 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.9 MiB/s wr, 36 op/s 2024-09-14T03:39:52.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:52 smithi195 ceph-mon[30695]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:39:52.932 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:52 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-14T03:39:52.932 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:52 smithi110 systemd[1]: Stopped Ceph mgr.y for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:52.932 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:52 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service: Consumed 40.402s CPU time. 2024-09-14T03:39:52.932 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:52 smithi110 systemd[1]: Starting Ceph mgr.y for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:39:53.047 INFO:tasks.rados.rados.0.smithi110.stdout:762: done (2 left) 2024-09-14T03:39:53.047 INFO:tasks.rados.rados.0.smithi110.stdout:763: copy_from oid 46 from oid 26 current snap is 72 2024-09-14T03:39:53.047 INFO:tasks.rados.rados.0.smithi110.stdout:764: copy_from oid 44 from oid 42 current snap is 72 2024-09-14T03:39:53.047 INFO:tasks.rados.rados.0.smithi110.stdout:765: setattr oid 22 current snap is 72 2024-09-14T03:39:53.047 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 390 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:39:53.048 INFO:tasks.rados.rados.0.smithi110.stdout:759: done (4 left) 2024-09-14T03:39:53.048 INFO:tasks.rados.rados.0.smithi110.stdout:761: done (3 left) 2024-09-14T03:39:53.048 INFO:tasks.rados.rados.0.smithi110.stdout:766: rollback oid 8 current snap is 72 2024-09-14T03:39:53.048 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 8 to 71 2024-09-14T03:39:53.048 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 72 2024-09-14T03:39:53.049 INFO:tasks.rados.rados.0.smithi110.stdout:767: seq_num 249 ranges {0=573440,573440=794624,1368064=409600} 2024-09-14T03:39:53.052 INFO:tasks.rados.rados.0.smithi110.stdout:767: writing smithi11075562-30 from 0 to 573440 tid 1 2024-09-14T03:39:53.058 INFO:tasks.rados.rados.0.smithi110.stdout:767: writing smithi11075562-30 from 573440 to 1368064 tid 2 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:767: writing smithi11075562-30 from 1368064 to 1777664 tid 3 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:768: rollback oid 9 current snap is 72 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 9 to 69 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:769: read oid 35 snap -1 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:769: expect (ObjNum 246 snap 69 seq_num 246) 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:770: read oid 49 snap -1 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:770: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 72 2024-09-14T03:39:53.062 INFO:tasks.rados.rados.0.smithi110.stdout:771: seq_num 250 ranges {0=589824,589824=679936,1269760=409600,1679360=114688} 2024-09-14T03:39:53.066 INFO:tasks.rados.rados.0.smithi110.stdout:771: writing smithi11075562-5 from 0 to 589824 tid 1 2024-09-14T03:39:53.070 INFO:tasks.rados.rados.0.smithi110.stdout:771: writing smithi11075562-5 from 589824 to 1269760 tid 2 2024-09-14T03:39:53.073 INFO:tasks.rados.rados.0.smithi110.stdout:771: writing smithi11075562-5 from 1269760 to 1679360 tid 3 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:771: writing smithi11075562-5 from 1679360 to 1794048 tid 4 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:772: read oid 34 snap 53 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:772: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:773: copy_from oid 38 from oid 31 current snap is 72 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 354 (ObjNum 163 snap 48 seq_num 163) dirty exists 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:766: finishing rollback tid 0 to smithi11075562-8 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:766: finishing rollback tid 1 to smithi11075562-8 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:766: finishing rollback tid 2 to smithi11075562-8 2024-09-14T03:39:53.075 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 358 (ObjNum 209 snap 62 seq_num 209) dirty exists 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:768: finishing rollback tid 0 to smithi11075562-9 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:768: finishing rollback tid 1 to smithi11075562-9 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:768: finishing rollback tid 2 to smithi11075562-9 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 272 (ObjNum 207 snap 62 seq_num 207) dirty exists 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:765: done (10 left) 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:766: done (9 left) 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:768: done (8 left) 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:774: read oid 17 snap -1 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:774: expect deleted 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:775: read oid 45 snap -1 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:775: expect (ObjNum 232 snap 69 seq_num 232) 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:776: snap_create 2024-09-14T03:39:53.076 INFO:tasks.rados.rados.0.smithi110.stdout:764: finishing copy_from racing read to smithi11075562-44 2024-09-14T03:39:53.077 INFO:tasks.rados.rados.0.smithi110.stdout:767: finishing write tid 1 to smithi11075562-30 2024-09-14T03:39:53.077 INFO:tasks.rados.rados.0.smithi110.stdout:767: finishing write tid 2 to smithi11075562-30 2024-09-14T03:39:53.077 INFO:tasks.rados.rados.0.smithi110.stdout:767: finishing write tid 3 to smithi11075562-30 2024-09-14T03:39:53.077 INFO:tasks.rados.rados.0.smithi110.stdout:767: finishing write tid 4 to smithi11075562-30 2024-09-14T03:39:53.078 INFO:tasks.rados.rados.0.smithi110.stdout:773: finishing copy_from to smithi11075562-38 2024-09-14T03:39:53.078 INFO:tasks.rados.rados.0.smithi110.stdout:773: got expected ENOENT (src dne) 2024-09-14T03:39:53.078 INFO:tasks.rados.rados.0.smithi110.stdout:773: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:39:53.078 INFO:tasks.rados.rados.0.smithi110.stdout:764: finishing copy_from to smithi11075562-44 2024-09-14T03:39:53.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 375 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:39:53.078 INFO:tasks.rados.rados.0.smithi110.stdout:772: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:39:53.101 INFO:tasks.rados.rados.0.smithi110.stdout:767: finishing write tid 5 to smithi11075562-30 2024-09-14T03:39:53.101 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 362 (ObjNum 249 snap 72 seq_num 249) dirty exists 2024-09-14T03:39:53.101 INFO:tasks.rados.rados.0.smithi110.stdout:767: left oid 30 (ObjNum 249 snap 72 seq_num 249) 2024-09-14T03:39:53.102 INFO:tasks.rados.rados.0.smithi110.stdout:771: finishing write tid 1 to smithi11075562-5 2024-09-14T03:39:53.102 INFO:tasks.rados.rados.0.smithi110.stdout:771: finishing write tid 2 to smithi11075562-5 2024-09-14T03:39:53.102 INFO:tasks.rados.rados.0.smithi110.stdout:763: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:39:53.102 INFO:tasks.rados.rados.0.smithi110.stdout:771: finishing write tid 3 to smithi11075562-5 2024-09-14T03:39:53.102 INFO:tasks.rados.rados.0.smithi110.stdout:771: finishing write tid 4 to smithi11075562-5 2024-09-14T03:39:53.102 INFO:tasks.rados.rados.0.smithi110.stdout:771: finishing write tid 5 to smithi11075562-5 2024-09-14T03:39:53.102 INFO:tasks.rados.rados.0.smithi110.stdout:775: expect (ObjNum 232 snap 69 seq_num 232) 2024-09-14T03:39:53.177 INFO:tasks.rados.rados.0.smithi110.stdout:771: finishing write tid 6 to smithi11075562-5 2024-09-14T03:39:53.177 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 456 (ObjNum 250 snap 72 seq_num 250) dirty exists 2024-09-14T03:39:53.177 INFO:tasks.rados.rados.0.smithi110.stdout:771: left oid 5 (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:39:53.177 INFO:tasks.rados.rados.0.smithi110.stdout:763: finishing copy_from to smithi11075562-46 2024-09-14T03:39:53.177 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 134 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:39:53.178 INFO:tasks.rados.rados.0.smithi110.stdout:769: expect (ObjNum 246 snap 69 seq_num 246) 2024-09-14T03:39:53.191 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:52 smithi110 podman[78660]: 2024-09-14 03:39:52.833063873 +0000 UTC m=+0.015764094 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:39:53.191 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:52 smithi110 podman[78660]: 2024-09-14 03:39:52.95456035 +0000 UTC m=+0.137260578 container create f1f945bddfa635ec5c24e58255d0568ab8cee6cfeb5b56de31fbef73e280ccb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T03:39:53.191 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:53 smithi110 podman[78660]: 2024-09-14 03:39:53.190655172 +0000 UTC m=+0.373355393 container init f1f945bddfa635ec5c24e58255d0568ab8cee6cfeb5b56de31fbef73e280ccb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:39:53.336 INFO:tasks.rados.rados.0.smithi110.stdout:770: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:39:53.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:53 smithi110 podman[78660]: 2024-09-14 03:39:53.193564115 +0000 UTC m=+0.376264336 container start f1f945bddfa635ec5c24e58255d0568ab8cee6cfeb5b56de31fbef73e280ccb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-a278231) 2024-09-14T03:39:53.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:53 smithi110 ceph-mgr[78678]: -- 172.21.15.110:0/1383671382 <== mon.2 v2:172.21.15.110:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b327baf0e0 con 0x55b327b8d400 2024-09-14T03:39:53.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:53 smithi110 bash[78660]: f1f945bddfa635ec5c24e58255d0568ab8cee6cfeb5b56de31fbef73e280ccb1 2024-09-14T03:39:53.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:53 smithi110 systemd[1]: Started Ceph mgr.y for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:763: done (10 left) 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:764: done (9 left) 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:767: done (8 left) 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:769: done (7 left) 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:770: done (6 left) 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:771: done (5 left) 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:772: done (4 left) 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:773: done (3 left) 2024-09-14T03:39:54.051 INFO:tasks.rados.rados.0.smithi110.stdout:774: done (2 left) 2024-09-14T03:39:54.052 INFO:tasks.rados.rados.0.smithi110.stdout:775: done (1 left) 2024-09-14T03:39:54.052 INFO:tasks.rados.rados.0.smithi110.stdout:776: done (0 left) 2024-09-14T03:39:54.052 INFO:tasks.rados.rados.0.smithi110.stdout:777: delete oid 9 current snap is 73 2024-09-14T03:39:54.054 INFO:tasks.rados.rados.0.smithi110.stdout:777: done (0 left) 2024-09-14T03:39:54.054 INFO:tasks.rados.rados.0.smithi110.stdout:778: snap_create 2024-09-14T03:39:54.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: pgmap v84: 121 pgs: 121 active+clean; 206 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.9 MiB/s wr, 36 op/s 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:39:54.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: pgmap v84: 121 pgs: 121 active+clean; 206 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.9 MiB/s wr, 36 op/s 2024-09-14T03:39:54.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:54.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:54 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:39:54] "GET /metrics HTTP/1.1" 200 36151 "" "Prometheus/2.51.0" 2024-09-14T03:39:54.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:39:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:39:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: pgmap v84: 121 pgs: 121 active+clean; 206 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.9 MiB/s wr, 36 op/s 2024-09-14T03:39:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:54.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:54.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:54 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:55.042 INFO:tasks.rados.rados.0.smithi110.stdout:778: done (0 left) 2024-09-14T03:39:55.042 INFO:tasks.rados.rados.0.smithi110.stdout:779: rmattr oid 13 current snap is 74 2024-09-14T03:39:55.042 INFO:tasks.rados.rados.0.smithi110.stdout:780: snap_create 2024-09-14T03:39:55.048 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 416 (ObjNum 237 snap 69 seq_num 237) dirty exists 2024-09-14T03:39:55.193 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:55 smithi110 ceph-mon[35374]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:39:55.193 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:55 smithi110 ceph-mon[35374]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:39:55.194 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:55 smithi110 ceph-mon[27365]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:39:55.194 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:55 smithi110 ceph-mon[27365]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:39:55.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:55 smithi195 ceph-mon[30695]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:39:55.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:55 smithi195 ceph-mon[30695]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:39:55.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:39:55.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:39:56.042 INFO:tasks.rados.rados.0.smithi110.stdout:779: done (1 left) 2024-09-14T03:39:56.043 INFO:tasks.rados.rados.0.smithi110.stdout:780: done (0 left) 2024-09-14T03:39:56.043 INFO:tasks.rados.rados.0.smithi110.stdout:781: copy_from oid 43 from oid 39 current snap is 75 2024-09-14T03:39:56.043 INFO:tasks.rados.rados.0.smithi110.stdout:782: delete oid 49 current snap is 75 2024-09-14T03:39:56.046 INFO:tasks.rados.rados.0.smithi110.stdout:782: done (1 left) 2024-09-14T03:39:56.047 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 75 2024-09-14T03:39:56.047 INFO:tasks.rados.rados.0.smithi110.stdout:783: seq_num 253 ranges {0=647168,647168=786432,1433600=417792} 2024-09-14T03:39:56.051 INFO:tasks.rados.rados.0.smithi110.stdout:783: writing smithi11075562-23 from 0 to 647168 tid 1 2024-09-14T03:39:56.057 INFO:tasks.rados.rados.0.smithi110.stdout:783: writing smithi11075562-23 from 647168 to 1433600 tid 2 2024-09-14T03:39:56.061 INFO:tasks.rados.rados.0.smithi110.stdout:783: writing smithi11075562-23 from 1433600 to 1851392 tid 3 2024-09-14T03:39:56.061 INFO:tasks.rados.rados.0.smithi110.stdout:784: rollback oid 3 current snap is 75 2024-09-14T03:39:56.061 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 3 to 66 2024-09-14T03:39:56.061 INFO:tasks.rados.rados.0.smithi110.stdout:783: finishing write tid 1 to smithi11075562-23 2024-09-14T03:39:56.061 INFO:tasks.rados.rados.0.smithi110.stdout:781: finishing copy_from racing read to smithi11075562-43 2024-09-14T03:39:56.061 INFO:tasks.rados.rados.0.smithi110.stdout:785: snap_create 2024-09-14T03:39:56.063 INFO:tasks.rados.rados.0.smithi110.stdout:783: finishing write tid 2 to smithi11075562-23 2024-09-14T03:39:56.064 INFO:tasks.rados.rados.0.smithi110.stdout:784: finishing rollback tid 1 to smithi11075562-3 2024-09-14T03:39:56.064 INFO:tasks.rados.rados.0.smithi110.stdout:783: finishing write tid 3 to smithi11075562-23 2024-09-14T03:39:56.064 INFO:tasks.rados.rados.0.smithi110.stdout:781: finishing copy_from to smithi11075562-43 2024-09-14T03:39:56.064 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 356 (ObjNum 208 snap 62 seq_num 208) dirty exists 2024-09-14T03:39:56.064 INFO:tasks.rados.rados.0.smithi110.stdout:784: finishing rollback tid 2 to smithi11075562-3 2024-09-14T03:39:56.064 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 272 (ObjNum 115 snap 28 seq_num 115) dirty exists 2024-09-14T03:39:56.064 INFO:tasks.rados.rados.0.smithi110.stdout:783: finishing write tid 4 to smithi11075562-23 2024-09-14T03:39:56.065 INFO:tasks.rados.rados.0.smithi110.stdout:783: finishing write tid 5 to smithi11075562-23 2024-09-14T03:39:56.065 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 241 (ObjNum 253 snap 75 seq_num 253) dirty exists 2024-09-14T03:39:56.065 INFO:tasks.rados.rados.0.smithi110.stdout:783: left oid 23 (ObjNum 253 snap 75 seq_num 253) 2024-09-14T03:39:56.302 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:56 smithi110 ceph-mon[35374]: pgmap v87: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:39:56.302 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:56 smithi110 ceph-mon[35374]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:39:56.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:56 smithi110 ceph-mon[27365]: pgmap v87: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:39:56.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:56 smithi110 ceph-mon[27365]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:39:56.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:56 smithi195 ceph-mon[30695]: pgmap v87: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:39:56.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:56 smithi195 ceph-mon[30695]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:39:57.055 INFO:tasks.rados.rados.0.smithi110.stdout:781: done (3 left) 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:783: done (2 left) 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:784: done (1 left) 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:785: done (0 left) 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:786: read oid 42 snap -1 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:786: expect (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:787: read oid 24 snap -1 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:787: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:788: copy_from oid 25 from oid 5 current snap is 76 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 76 2024-09-14T03:39:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:789: seq_num 254 ranges {5496832=663552,6160384=663552,6823936=540672} 2024-09-14T03:39:57.091 INFO:tasks.rados.rados.0.smithi110.stdout:789: writing smithi11075562-13 from 5496832 to 6160384 tid 1 2024-09-14T03:39:57.096 INFO:tasks.rados.rados.0.smithi110.stdout:789: writing smithi11075562-13 from 6160384 to 6823936 tid 2 2024-09-14T03:39:57.100 INFO:tasks.rados.rados.0.smithi110.stdout:789: writing smithi11075562-13 from 6823936 to 7364608 tid 3 2024-09-14T03:39:57.101 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 76 2024-09-14T03:39:57.101 INFO:tasks.rados.rados.0.smithi110.stdout:790: seq_num 255 ranges {1277952=679936,1957888=548864,2506752=598016,3104768=57344} 2024-09-14T03:39:57.111 INFO:tasks.rados.rados.0.smithi110.stdout:790: writing smithi11075562-4 from 1277952 to 1957888 tid 1 2024-09-14T03:39:57.115 INFO:tasks.rados.rados.0.smithi110.stdout:790: writing smithi11075562-4 from 1957888 to 2506752 tid 2 2024-09-14T03:39:57.118 INFO:tasks.rados.rados.0.smithi110.stdout:790: writing smithi11075562-4 from 2506752 to 3104768 tid 3 2024-09-14T03:39:57.119 INFO:tasks.rados.rados.0.smithi110.stdout:790: writing smithi11075562-4 from 3104768 to 3162112 tid 4 2024-09-14T03:39:57.119 INFO:tasks.rados.rados.0.smithi110.stdout:791: rmattr oid 12 current snap is 76 2024-09-14T03:39:57.119 INFO:tasks.rados.rados.0.smithi110.stdout:791: done (5 left) 2024-09-14T03:39:57.119 INFO:tasks.rados.rados.0.smithi110.stdout:792: read oid 39 snap -1 2024-09-14T03:39:57.119 INFO:tasks.rados.rados.0.smithi110.stdout:792: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:39:57.119 INFO:tasks.rados.rados.0.smithi110.stdout:793: copy_from oid 30 from oid 34 current snap is 76 2024-09-14T03:39:57.119 INFO:tasks.rados.rados.0.smithi110.stdout:787: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-14T03:39:57.166 INFO:tasks.rados.rados.0.smithi110.stdout:788: finishing copy_from racing read to smithi11075562-25 2024-09-14T03:39:57.166 INFO:tasks.rados.rados.0.smithi110.stdout:786: expect (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:39:57.208 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:39:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:39:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:39:57.256 INFO:tasks.rados.rados.0.smithi110.stdout:788: finishing copy_from to smithi11075562-25 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 456 (ObjNum 250 snap 72 seq_num 250) dirty exists 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:789: finishing write tid 1 to smithi11075562-13 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:789: finishing write tid 2 to smithi11075562-13 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:789: finishing write tid 3 to smithi11075562-13 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:789: finishing write tid 4 to smithi11075562-13 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:786: done (6 left) 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:787: done (5 left) 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:788: done (4 left) 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:794: read oid 31 snap -1 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:794: expect deleted 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:795: snap_create 2024-09-14T03:39:57.257 INFO:tasks.rados.rados.0.smithi110.stdout:789: finishing write tid 5 to smithi11075562-13 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 423 (ObjNum 254 snap 76 seq_num 254) dirty exists 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:789: left oid 13 (ObjNum 254 snap 76 seq_num 254) 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:790: finishing write tid 1 to smithi11075562-4 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:790: finishing write tid 2 to smithi11075562-4 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:790: finishing write tid 3 to smithi11075562-4 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:790: finishing write tid 4 to smithi11075562-4 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:790: finishing write tid 5 to smithi11075562-4 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:790: finishing write tid 6 to smithi11075562-4 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 303 (ObjNum 255 snap 76 seq_num 255) dirty exists 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:790: left oid 4 (ObjNum 255 snap 76 seq_num 255) 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:793: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:793: finishing copy_from to smithi11075562-30 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 71 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:39:57.258 INFO:tasks.rados.rados.0.smithi110.stdout:792: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:39:58.061 INFO:tasks.rados.rados.0.smithi110.stdout:789: done (5 left) 2024-09-14T03:39:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:790: done (4 left) 2024-09-14T03:39:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:792: done (3 left) 2024-09-14T03:39:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:793: done (2 left) 2024-09-14T03:39:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:794: done (1 left) 2024-09-14T03:39:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:795: done (0 left) 2024-09-14T03:39:58.062 INFO:tasks.rados.rados.0.smithi110.stdout:796: snap_create 2024-09-14T03:39:58.339 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:58 smithi195 ceph-mon[30695]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:39:58.339 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:58 smithi195 ceph-mon[30695]: pgmap v90: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:39:58.339 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:58 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.339 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:58 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.339 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:58 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.339 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:58 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[27365]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[27365]: pgmap v90: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[35374]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[35374]: pgmap v90: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:39:58.656 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:39:58] ENGINE Bus STARTING 2024-09-14T03:39:58.657 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: CherryPy Checker: 2024-09-14T03:39:58.657 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: The Application mounted at '' has an empty config. 2024-09-14T03:39:58.657 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:39:58.657 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:39:58] ENGINE Serving on http://:::9283 2024-09-14T03:39:58.657 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:39:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:39:58] ENGINE Bus STARTED 2024-09-14T03:39:59.061 INFO:tasks.rados.rados.0.smithi110.stdout:796: done (0 left) 2024-09-14T03:39:59.061 INFO:tasks.rados.rados.0.smithi110.stdout:797: copy_from oid 22 from oid 47 current snap is 78 2024-09-14T03:39:59.061 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 78 2024-09-14T03:39:59.062 INFO:tasks.rados.rados.0.smithi110.stdout:798: seq_num 256 ranges {0=696320,696320=434176,1130496=770048} 2024-09-14T03:39:59.065 INFO:tasks.rados.rados.0.smithi110.stdout:798: writing smithi11075562-27 from 0 to 696320 tid 1 2024-09-14T03:39:59.068 INFO:tasks.rados.rados.0.smithi110.stdout:798: writing smithi11075562-27 from 696320 to 1130496 tid 2 2024-09-14T03:39:59.074 INFO:tasks.rados.rados.0.smithi110.stdout:798: writing smithi11075562-27 from 1130496 to 1900544 tid 3 2024-09-14T03:39:59.074 INFO:tasks.rados.rados.0.smithi110.stdout:799: snap_create 2024-09-14T03:39:59.074 INFO:tasks.rados.rados.0.smithi110.stdout:798: finishing write tid 1 to smithi11075562-27 2024-09-14T03:39:59.074 INFO:tasks.rados.rados.0.smithi110.stdout:798: finishing write tid 2 to smithi11075562-27 2024-09-14T03:39:59.077 INFO:tasks.rados.rados.0.smithi110.stdout:798: finishing write tid 3 to smithi11075562-27 2024-09-14T03:39:59.077 INFO:tasks.rados.rados.0.smithi110.stdout:798: finishing write tid 4 to smithi11075562-27 2024-09-14T03:39:59.078 INFO:tasks.rados.rados.0.smithi110.stdout:798: finishing write tid 5 to smithi11075562-27 2024-09-14T03:39:59.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 93 (ObjNum 256 snap 78 seq_num 256) dirty exists 2024-09-14T03:39:59.078 INFO:tasks.rados.rados.0.smithi110.stdout:798: left oid 27 (ObjNum 256 snap 78 seq_num 256) 2024-09-14T03:39:59.080 INFO:tasks.rados.rados.0.smithi110.stdout:797: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:39:59.081 INFO:tasks.rados.rados.0.smithi110.stdout:797: finishing copy_from to smithi11075562-22 2024-09-14T03:39:59.081 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 400 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:39:59.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:59 smithi195 ceph-mon[30695]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:39:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:59 smithi195 ceph-mon[30695]: Standby manager daemon y restarted 2024-09-14T03:39:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:59 smithi195 ceph-mon[30695]: Standby manager daemon y started 2024-09-14T03:39:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:59 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:39:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:59 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:39:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:59 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:39:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:39:59 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:39:59.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[35374]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:39:59.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[35374]: Standby manager daemon y restarted 2024-09-14T03:39:59.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[35374]: Standby manager daemon y started 2024-09-14T03:39:59.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:39:59.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[27365]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[27365]: Standby manager daemon y restarted 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[27365]: Standby manager daemon y started 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:39:59.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:39:59 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:00.043 INFO:tasks.rados.rados.0.smithi110.stdout:797: done (2 left) 2024-09-14T03:40:00.044 INFO:tasks.rados.rados.0.smithi110.stdout:798: done (1 left) 2024-09-14T03:40:00.044 INFO:tasks.rados.rados.0.smithi110.stdout:799: done (0 left) 2024-09-14T03:40:00.044 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 79 2024-09-14T03:40:00.044 INFO:tasks.rados.rados.0.smithi110.stdout:800: seq_num 257 ranges {4014080=712704,4726784=720896,5447680=483328} 2024-09-14T03:40:00.069 INFO:tasks.rados.rados.0.smithi110.stdout:800: writing smithi11075562-35 from 4014080 to 4726784 tid 1 2024-09-14T03:40:00.074 INFO:tasks.rados.rados.0.smithi110.stdout:800: writing smithi11075562-35 from 4726784 to 5447680 tid 2 2024-09-14T03:40:00.077 INFO:tasks.rados.rados.0.smithi110.stdout:800: writing smithi11075562-35 from 5447680 to 5931008 tid 3 2024-09-14T03:40:00.077 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 79 2024-09-14T03:40:00.077 INFO:tasks.rados.rados.0.smithi110.stdout:800: finishing write tid 1 to smithi11075562-35 2024-09-14T03:40:00.077 INFO:tasks.rados.rados.0.smithi110.stdout:801: seq_num 258 ranges {0=729088,729088=606208,1335296=598016} 2024-09-14T03:40:00.080 INFO:tasks.rados.rados.0.smithi110.stdout:801: writing smithi11075562-40 from 0 to 729088 tid 1 2024-09-14T03:40:00.084 INFO:tasks.rados.rados.0.smithi110.stdout:801: writing smithi11075562-40 from 729088 to 1335296 tid 2 2024-09-14T03:40:00.088 INFO:tasks.rados.rados.0.smithi110.stdout:801: writing smithi11075562-40 from 1335296 to 1933312 tid 3 2024-09-14T03:40:00.088 INFO:tasks.rados.rados.0.smithi110.stdout:802: read oid 22 snap -1 2024-09-14T03:40:00.088 INFO:tasks.rados.rados.0.smithi110.stdout:802: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:40:00.088 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 79 2024-09-14T03:40:00.088 INFO:tasks.rados.rados.0.smithi110.stdout:800: finishing write tid 2 to smithi11075562-35 2024-09-14T03:40:00.088 INFO:tasks.rados.rados.0.smithi110.stdout:800: finishing write tid 3 to smithi11075562-35 2024-09-14T03:40:00.088 INFO:tasks.rados.rados.0.smithi110.stdout:803: seq_num 259 ranges {0=745472,745472=540672,1286144=581632,1867776=81920} 2024-09-14T03:40:00.092 INFO:tasks.rados.rados.0.smithi110.stdout:803: writing smithi11075562-2 from 0 to 745472 tid 1 2024-09-14T03:40:00.096 INFO:tasks.rados.rados.0.smithi110.stdout:803: writing smithi11075562-2 from 745472 to 1286144 tid 2 2024-09-14T03:40:00.099 INFO:tasks.rados.rados.0.smithi110.stdout:803: writing smithi11075562-2 from 1286144 to 1867776 tid 3 2024-09-14T03:40:00.100 INFO:tasks.rados.rados.0.smithi110.stdout:803: writing smithi11075562-2 from 1867776 to 1949696 tid 4 2024-09-14T03:40:00.100 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 79 2024-09-14T03:40:00.100 INFO:tasks.rados.rados.0.smithi110.stdout:800: finishing write tid 4 to smithi11075562-35 2024-09-14T03:40:00.101 INFO:tasks.rados.rados.0.smithi110.stdout:800: finishing write tid 5 to smithi11075562-35 2024-09-14T03:40:00.101 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 428 (ObjNum 257 snap 79 seq_num 257) dirty exists 2024-09-14T03:40:00.101 INFO:tasks.rados.rados.0.smithi110.stdout:800: left oid 35 (ObjNum 257 snap 79 seq_num 257) 2024-09-14T03:40:00.101 INFO:tasks.rados.rados.0.smithi110.stdout:801: finishing write tid 1 to smithi11075562-40 2024-09-14T03:40:00.101 INFO:tasks.rados.rados.0.smithi110.stdout:804: seq_num 260 ranges {4046848=761856,4808704=417792,5226496=786432} 2024-09-14T03:40:00.127 INFO:tasks.rados.rados.0.smithi110.stdout:804: writing smithi11075562-39 from 4046848 to 4808704 tid 1 2024-09-14T03:40:00.129 INFO:tasks.rados.rados.0.smithi110.stdout:804: writing smithi11075562-39 from 4808704 to 5226496 tid 2 2024-09-14T03:40:00.134 INFO:tasks.rados.rados.0.smithi110.stdout:804: writing smithi11075562-39 from 5226496 to 6012928 tid 3 2024-09-14T03:40:00.134 INFO:tasks.rados.rados.0.smithi110.stdout:800: done (4 left) 2024-09-14T03:40:00.134 INFO:tasks.rados.rados.0.smithi110.stdout:805: rmattr oid 27 current snap is 79 2024-09-14T03:40:00.134 INFO:tasks.rados.rados.0.smithi110.stdout:805: done (4 left) 2024-09-14T03:40:00.135 INFO:tasks.rados.rados.0.smithi110.stdout:806: snap_remove snap 51 2024-09-14T03:40:00.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[35374]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:40:00.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[35374]: mgrmap e30: x(active, since 59s), standbys: y 2024-09-14T03:40:00.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[35374]: pgmap v93: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:00.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:00.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:00.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[35374]: overall HEALTH_OK 2024-09-14T03:40:00.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[35374]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:40:00.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[27365]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:40:00.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[27365]: mgrmap e30: x(active, since 59s), standbys: y 2024-09-14T03:40:00.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[27365]: pgmap v93: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:00.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:00.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:00.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[27365]: overall HEALTH_OK 2024-09-14T03:40:00.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:00 smithi110 ceph-mon[27365]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:40:00.458 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:00 smithi195 ceph-mon[30695]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:40:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:00 smithi195 ceph-mon[30695]: mgrmap e30: x(active, since 59s), standbys: y 2024-09-14T03:40:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:00 smithi195 ceph-mon[30695]: pgmap v93: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:00 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:00 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:00 smithi195 ceph-mon[30695]: overall HEALTH_OK 2024-09-14T03:40:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:00 smithi195 ceph-mon[30695]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:40:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:801: finishing write tid 2 to smithi11075562-40 2024-09-14T03:40:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:801: finishing write tid 3 to smithi11075562-40 2024-09-14T03:40:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:801: finishing write tid 4 to smithi11075562-40 2024-09-14T03:40:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:801: finishing write tid 5 to smithi11075562-40 2024-09-14T03:40:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 355 (ObjNum 258 snap 79 seq_num 258) dirty exists 2024-09-14T03:40:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:801: left oid 40 (ObjNum 258 snap 79 seq_num 258) 2024-09-14T03:40:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:802: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:803: finishing write tid 1 to smithi11075562-2 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:803: finishing write tid 2 to smithi11075562-2 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:803: finishing write tid 3 to smithi11075562-2 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:803: finishing write tid 4 to smithi11075562-2 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:803: finishing write tid 5 to smithi11075562-2 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:801: done (4 left) 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:802: done (3 left) 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:806: done (2 left) 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 79 2024-09-14T03:40:01.129 INFO:tasks.rados.rados.0.smithi110.stdout:803: finishing write tid 6 to smithi11075562-2 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 277 (ObjNum 259 snap 79 seq_num 259) dirty exists 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:803: left oid 2 (ObjNum 259 snap 79 seq_num 259) 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:804: finishing write tid 1 to smithi11075562-39 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:804: finishing write tid 2 to smithi11075562-39 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:804: finishing write tid 3 to smithi11075562-39 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:804: finishing write tid 4 to smithi11075562-39 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:804: finishing write tid 5 to smithi11075562-39 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 413 (ObjNum 260 snap 79 seq_num 260) dirty exists 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:804: left oid 39 (ObjNum 260 snap 79 seq_num 260) 2024-09-14T03:40:01.130 INFO:tasks.rados.rados.0.smithi110.stdout:807: seq_num 261 ranges {0=778240,778240=704512,1482752=499712} 2024-09-14T03:40:01.132 INFO:tasks.rados.rados.0.smithi110.stdout:807: writing smithi11075562-50 from 0 to 778240 tid 1 2024-09-14T03:40:01.137 INFO:tasks.rados.rados.0.smithi110.stdout:807: writing smithi11075562-50 from 778240 to 1482752 tid 2 2024-09-14T03:40:01.140 INFO:tasks.rados.rados.0.smithi110.stdout:807: writing smithi11075562-50 from 1482752 to 1982464 tid 3 2024-09-14T03:40:01.140 INFO:tasks.rados.rados.0.smithi110.stdout:803: done (2 left) 2024-09-14T03:40:01.140 INFO:tasks.rados.rados.0.smithi110.stdout:804: done (1 left) 2024-09-14T03:40:01.140 INFO:tasks.rados.rados.0.smithi110.stdout:808: snap_create 2024-09-14T03:40:01.140 INFO:tasks.rados.rados.0.smithi110.stdout:807: finishing write tid 1 to smithi11075562-50 2024-09-14T03:40:01.140 INFO:tasks.rados.rados.0.smithi110.stdout:807: finishing write tid 2 to smithi11075562-50 2024-09-14T03:40:01.142 INFO:tasks.rados.rados.0.smithi110.stdout:807: finishing write tid 3 to smithi11075562-50 2024-09-14T03:40:01.142 INFO:tasks.rados.rados.0.smithi110.stdout:807: finishing write tid 4 to smithi11075562-50 2024-09-14T03:40:01.143 INFO:tasks.rados.rados.0.smithi110.stdout:807: finishing write tid 5 to smithi11075562-50 2024-09-14T03:40:01.143 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 78 (ObjNum 261 snap 79 seq_num 261) dirty exists 2024-09-14T03:40:01.143 INFO:tasks.rados.rados.0.smithi110.stdout:807: left oid 50 (ObjNum 261 snap 79 seq_num 261) 2024-09-14T03:40:01.262 INFO:tasks.rados.rados.0.smithi110.stdout:807: done (1 left) 2024-09-14T03:40:01.262 INFO:tasks.rados.rados.0.smithi110.stdout:808: done (0 left) 2024-09-14T03:40:01.262 INFO:tasks.rados.rados.0.smithi110.stdout:809: delete oid 46 current snap is 80 2024-09-14T03:40:01.268 INFO:tasks.rados.rados.0.smithi110.stdout:809: done (0 left) 2024-09-14T03:40:01.268 INFO:tasks.rados.rados.0.smithi110.stdout:810: delete oid 44 current snap is 80 2024-09-14T03:40:01.270 INFO:tasks.rados.rados.0.smithi110.stdout:810: done (0 left) 2024-09-14T03:40:01.270 INFO:tasks.rados.rados.0.smithi110.stdout:811: read oid 46 snap -1 2024-09-14T03:40:01.271 INFO:tasks.rados.rados.0.smithi110.stdout:811: expect deleted 2024-09-14T03:40:01.271 INFO:tasks.rados.rados.0.smithi110.stdout:812: rollback oid 47 current snap is 80 2024-09-14T03:40:01.271 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 47 to 71 2024-09-14T03:40:01.271 INFO:tasks.rados.rados.0.smithi110.stdout:813: read oid 43 snap -1 2024-09-14T03:40:01.271 INFO:tasks.rados.rados.0.smithi110.stdout:813: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:40:01.271 INFO:tasks.rados.rados.0.smithi110.stdout:814: snap_create 2024-09-14T03:40:01.273 INFO:tasks.rados.rados.0.smithi110.stdout:812: finishing rollback tid 0 to smithi11075562-47 2024-09-14T03:40:01.274 INFO:tasks.rados.rados.0.smithi110.stdout:812: finishing rollback tid 1 to smithi11075562-47 2024-09-14T03:40:01.274 INFO:tasks.rados.rados.0.smithi110.stdout:812: finishing rollback tid 2 to smithi11075562-47 2024-09-14T03:40:01.275 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 472 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:40:01.282 INFO:tasks.rados.rados.0.smithi110.stdout:813: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:40:01.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:01 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:01.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:01 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:01.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:01] ENGINE Bus STOPPING 2024-09-14T03:40:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:01 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:01.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:40:01.261+0000 7fed3d8d1640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-14T03:40:01.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ignoring --setuser ceph since I am not root 2024-09-14T03:40:01.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: ignoring --setgroup ceph since I am not root 2024-09-14T03:40:01.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:01 smithi195 ceph-mgr[63999]: -- 172.21.15.195:0/956539734 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5622050150e0 con 0x562204ff3000 2024-09-14T03:40:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:40:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:01] ENGINE Bus STOPPED 2024-09-14T03:40:02.263 INFO:tasks.rados.rados.0.smithi110.stdout:811: done (3 left) 2024-09-14T03:40:02.263 INFO:tasks.rados.rados.0.smithi110.stdout:812: done (2 left) 2024-09-14T03:40:02.263 INFO:tasks.rados.rados.0.smithi110.stdout:813: done (1 left) 2024-09-14T03:40:02.263 INFO:tasks.rados.rados.0.smithi110.stdout:814: done (0 left) 2024-09-14T03:40:02.263 INFO:tasks.rados.rados.0.smithi110.stdout:815: rollback oid 27 current snap is 81 2024-09-14T03:40:02.263 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 53 2024-09-14T03:40:02.263 INFO:tasks.rados.rados.0.smithi110.stdout:816: snap_remove snap 63 2024-09-14T03:40:02.363 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbf9d4e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.363 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbf9d4e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.363 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbf9d4e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.363 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbf9d4e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbf9d4e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbfd555640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbfd555640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbfd555640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbfd555640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:02.070+0000 7efbfd555640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:02] ENGINE Bus STARTING 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:40:02.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: Activating manager daemon y 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:40:02.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.24679 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: mgrmap e31: y(active, starting, since 0.00663176s) 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:02.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:40:02.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: Activating manager daemon y 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:40:02.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.24679 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: mgrmap e31: y(active, starting, since 0.00663176s) 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:40:02.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: Manager daemon y is now available 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:40:02.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:40:02.371 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: Manager daemon y is now available 2024-09-14T03:40:02.371 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:02.371 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:02.371 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:02.371 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:40:02.371 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:40:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:40:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 172.21.15.195:0/2183327445' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: Activating manager daemon y 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.24679 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: mgrmap e31: y(active, starting, since 0.00663176s) 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:02.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:40:02.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: Manager daemon y is now available 2024-09-14T03:40:02.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:02.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:02.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:02.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:40:02.462 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:02 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:40:02.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:02] ENGINE Serving on http://:::9283 2024-09-14T03:40:02.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:02] ENGINE Bus STARTED 2024-09-14T03:40:03.275 INFO:tasks.rados.rados.0.smithi110.stdout:816: done (1 left) 2024-09-14T03:40:03.275 INFO:tasks.rados.rados.0.smithi110.stdout:817: copy_from oid 11 from oid 41 current snap is 81 2024-09-14T03:40:03.275 INFO:tasks.rados.rados.0.smithi110.stdout:815: finishing rollback tid 0 to smithi11075562-27 2024-09-14T03:40:03.275 INFO:tasks.rados.rados.0.smithi110.stdout:815: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:40:03.275 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 95 (ObjNum 129 snap 0 seq_num 65497584) dirty dne 2024-09-14T03:40:03.275 INFO:tasks.rados.rados.0.smithi110.stdout:815: done (1 left) 2024-09-14T03:40:03.275 INFO:tasks.rados.rados.0.smithi110.stdout:818: copy_from oid 46 from oid 17 current snap is 81 2024-09-14T03:40:03.275 INFO:tasks.rados.rados.0.smithi110.stdout:819: delete oid 29 current snap is 81 2024-09-14T03:40:03.281 INFO:tasks.rados.rados.0.smithi110.stdout:818: finishing copy_from to smithi11075562-46 2024-09-14T03:40:03.282 INFO:tasks.rados.rados.0.smithi110.stdout:818: got expected ENOENT (src dne) 2024-09-14T03:40:03.282 INFO:tasks.rados.rados.0.smithi110.stdout:818: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:40:03.285 INFO:tasks.rados.rados.0.smithi110.stdout:818: done (2 left) 2024-09-14T03:40:03.285 INFO:tasks.rados.rados.0.smithi110.stdout:819: done (1 left) 2024-09-14T03:40:03.285 INFO:tasks.rados.rados.0.smithi110.stdout:820: snap_remove snap 71 2024-09-14T03:40:03.636 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-mon[35374]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:40:03.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-mon[35374]: mgrmap e32: y(active, since 1.01567s) 2024-09-14T03:40:03.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:03] ENGINE Bus STARTING 2024-09-14T03:40:03.637 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-mon[27365]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:40:03.637 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-mon[27365]: mgrmap e32: y(active, since 1.01567s) 2024-09-14T03:40:03.637 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-mon[27365]: [14/Sep/2024:03:40:03] ENGINE Bus STARTING 2024-09-14T03:40:03.637 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:03.223+0000 7efbe0adf640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:40:03] ENGINE Error in HTTPServer.serve 2024-09-14T03:40:03.637 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: Traceback (most recent call last): 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: self._connections.run(self.expiration_interval) 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: self._run(expiration_interval) 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: s = self.context.wrap_socket( 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: return self.sslsocket_class._create( 2024-09-14T03:40:03.638 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:40:03.639 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: self.do_handshake() 2024-09-14T03:40:03.639 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:40:03.639 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: self._sslobj.do_handshake() 2024-09-14T03:40:03.639 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:40:03.639 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:03 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:03.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:03 smithi195 ceph-mon[30695]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:40:03.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:03 smithi195 ceph-mon[30695]: mgrmap e32: y(active, since 1.01567s) 2024-09-14T03:40:03.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:03 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:03] ENGINE Bus STARTING 2024-09-14T03:40:04.279 INFO:tasks.rados.rados.0.smithi110.stdout:820: done (1 left) 2024-09-14T03:40:04.279 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 81 2024-09-14T03:40:04.279 INFO:tasks.rados.rados.0.smithi110.stdout:821: seq_num 265 ranges {1949696=442368,2392064=647168,3039232=573440,3612672=385024} 2024-09-14T03:40:04.292 INFO:tasks.rados.rados.0.smithi110.stdout:821: writing smithi11075562-2 from 1949696 to 2392064 tid 1 2024-09-14T03:40:04.296 INFO:tasks.rados.rados.0.smithi110.stdout:821: writing smithi11075562-2 from 2392064 to 3039232 tid 2 2024-09-14T03:40:04.300 INFO:tasks.rados.rados.0.smithi110.stdout:821: writing smithi11075562-2 from 3039232 to 3612672 tid 3 2024-09-14T03:40:04.302 INFO:tasks.rados.rados.0.smithi110.stdout:821: writing smithi11075562-2 from 3612672 to 3997696 tid 4 2024-09-14T03:40:04.302 INFO:tasks.rados.rados.0.smithi110.stdout:822: delete oid 5 current snap is 81 2024-09-14T03:40:04.302 INFO:tasks.rados.rados.0.smithi110.stdout:817: finishing copy_from racing read to smithi11075562-11 2024-09-14T03:40:04.302 INFO:tasks.rados.rados.0.smithi110.stdout:817: finishing copy_from to smithi11075562-11 2024-09-14T03:40:04.302 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 381 (ObjNum 238 snap 69 seq_num 238) dirty exists 2024-09-14T03:40:04.302 INFO:tasks.rados.rados.0.smithi110.stdout:821: finishing write tid 1 to smithi11075562-2 2024-09-14T03:40:04.303 INFO:tasks.rados.rados.0.smithi110.stdout:821: finishing write tid 2 to smithi11075562-2 2024-09-14T03:40:04.303 INFO:tasks.rados.rados.0.smithi110.stdout:821: finishing write tid 3 to smithi11075562-2 2024-09-14T03:40:04.304 INFO:tasks.rados.rados.0.smithi110.stdout:821: finishing write tid 4 to smithi11075562-2 2024-09-14T03:40:04.304 INFO:tasks.rados.rados.0.smithi110.stdout:817: done (2 left) 2024-09-14T03:40:04.304 INFO:tasks.rados.rados.0.smithi110.stdout:822: done (1 left) 2024-09-14T03:40:04.304 INFO:tasks.rados.rados.0.smithi110.stdout:823: snap_remove snap 70 2024-09-14T03:40:04.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:04 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:03] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:40:04.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:04 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:03] ENGINE Error in HTTPServer.serve 2024-09-14T03:40:04.577 INFO:journalctl@ceph.mon.b.smithi195.stdout: Traceback (most recent call last): 2024-09-14T03:40:04.577 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._run(expiration_interval) 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: s = self.context.wrap_socket( 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: return self.sslsocket_class._create( 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi195.stdout: self.do_handshake() 2024-09-14T03:40:04.579 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:40:04.579 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._sslobj.do_handshake() 2024-09-14T03:40:04.579 INFO:journalctl@ceph.mon.b.smithi195.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:40:04.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:04 smithi195 ceph-mon[30695]: pgmap v5: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:04.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:04 smithi195 ceph-mon[30695]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:40:04.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:04 smithi195 ceph-mon[30695]: mgrmap e33: y(active, since 2s) 2024-09-14T03:40:04.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:04 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:03] ENGINE Serving on http://172.21.15.110:8765 2024-09-14T03:40:04.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:04 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:03] ENGINE Bus STARTED 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:03] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:03] ENGINE Error in HTTPServer.serve 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:40:04.582 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout: self.do_handshake() 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[35374]: pgmap v5: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[35374]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[35374]: mgrmap e33: y(active, since 2s) 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:03] ENGINE Serving on http://172.21.15.110:8765 2024-09-14T03:40:04.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:03] ENGINE Bus STARTED 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[27365]: [14/Sep/2024:03:40:03] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[27365]: [14/Sep/2024:03:40:03] ENGINE Error in HTTPServer.serve 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:40:04.584 INFO:journalctl@ceph.mon.a.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: self.do_handshake() 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[27365]: pgmap v5: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[27365]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[27365]: mgrmap e33: y(active, since 2s) 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[27365]: [14/Sep/2024:03:40:03] ENGINE Serving on http://172.21.15.110:8765 2024-09-14T03:40:04.585 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:04 smithi110 ceph-mon[27365]: [14/Sep/2024:03:40:03] ENGINE Bus STARTED 2024-09-14T03:40:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:823: done (1 left) 2024-09-14T03:40:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:824: read oid 9 snap -1 2024-09-14T03:40:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:824: expect deleted 2024-09-14T03:40:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:825: snap_remove snap 73 2024-09-14T03:40:05.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:05 smithi110 ceph-mon[27365]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:40:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:05 smithi110 ceph-mon[27365]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:40:05.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:05 smithi110 ceph-mon[35374]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:40:05.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:05 smithi110 ceph-mon[35374]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:40:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:05 smithi195 ceph-mon[30695]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:40:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:05 smithi195 ceph-mon[30695]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:40:06.047 INFO:tasks.rados.rados.0.smithi110.stdout:825: done (2 left) 2024-09-14T03:40:06.047 INFO:tasks.rados.rados.0.smithi110.stdout:826: read oid 47 snap -1 2024-09-14T03:40:06.047 INFO:tasks.rados.rados.0.smithi110.stdout:826: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:40:06.047 INFO:tasks.rados.rados.0.smithi110.stdout:827: read oid 42 snap -1 2024-09-14T03:40:06.048 INFO:tasks.rados.rados.0.smithi110.stdout:827: expect (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:40:06.048 INFO:tasks.rados.rados.0.smithi110.stdout:828: delete oid 36 current snap is 81 2024-09-14T03:40:06.048 INFO:tasks.rados.rados.0.smithi110.stdout:821: finishing write tid 5 to smithi11075562-2 2024-09-14T03:40:06.048 INFO:tasks.rados.rados.0.smithi110.stdout:821: finishing write tid 6 to smithi11075562-2 2024-09-14T03:40:06.048 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 295 (ObjNum 265 snap 81 seq_num 265) dirty exists 2024-09-14T03:40:06.048 INFO:tasks.rados.rados.0.smithi110.stdout:821: left oid 2 (ObjNum 265 snap 81 seq_num 265) 2024-09-14T03:40:06.062 INFO:tasks.rados.rados.0.smithi110.stdout:826: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:40:06.108 INFO:tasks.rados.rados.0.smithi110.stdout:827: expect (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:40:06.202 INFO:tasks.rados.rados.0.smithi110.stdout:821: done (4 left) 2024-09-14T03:40:06.202 INFO:tasks.rados.rados.0.smithi110.stdout:824: done (3 left) 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:826: done (2 left) 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:827: done (1 left) 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:828: done (0 left) 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:829: read oid 32 snap -1 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:829: expect (ObjNum 211 snap 65 seq_num 211) 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:830: read oid 50 snap -1 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:830: expect (ObjNum 261 snap 79 seq_num 261) 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 81 2024-09-14T03:40:06.203 INFO:tasks.rados.rados.0.smithi110.stdout:831: seq_num 268 ranges {0=491520,491520=745472,1236992=745472,1982464=114688} 2024-09-14T03:40:06.204 INFO:tasks.rados.rados.0.smithi110.stdout:831: writing smithi11075562-10 from 0 to 491520 tid 1 2024-09-14T03:40:06.211 INFO:tasks.rados.rados.0.smithi110.stdout:831: writing smithi11075562-10 from 491520 to 1236992 tid 2 2024-09-14T03:40:06.215 INFO:tasks.rados.rados.0.smithi110.stdout:831: writing smithi11075562-10 from 1236992 to 1982464 tid 3 2024-09-14T03:40:06.216 INFO:tasks.rados.rados.0.smithi110.stdout:831: writing smithi11075562-10 from 1982464 to 2097152 tid 4 2024-09-14T03:40:06.216 INFO:tasks.rados.rados.0.smithi110.stdout:832: setattr oid 27 current snap is 81 2024-09-14T03:40:06.216 INFO:tasks.rados.rados.0.smithi110.stdout:830: expect (ObjNum 261 snap 79 seq_num 261) 2024-09-14T03:40:06.274 INFO:tasks.rados.rados.0.smithi110.stdout:831: finishing write tid 1 to smithi11075562-10 2024-09-14T03:40:06.274 INFO:tasks.rados.rados.0.smithi110.stdout:829: expect (ObjNum 211 snap 65 seq_num 211) 2024-09-14T03:40:06.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-mon[30695]: pgmap v9: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-mon[30695]: mgrmap e34: y(active, since 4s) 2024-09-14T03:40:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-mon[30695]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:40:06.497 INFO:tasks.rados.rados.0.smithi110.stdout:831: finishing write tid 2 to smithi11075562-10 2024-09-14T03:40:06.497 INFO:tasks.rados.rados.0.smithi110.stdout:831: finishing write tid 3 to smithi11075562-10 2024-09-14T03:40:06.497 INFO:tasks.rados.rados.0.smithi110.stdout:831: finishing write tid 4 to smithi11075562-10 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:831: finishing write tid 5 to smithi11075562-10 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:831: finishing write tid 6 to smithi11075562-10 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 246 (ObjNum 268 snap 81 seq_num 268) dirty exists 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:831: left oid 10 (ObjNum 268 snap 81 seq_num 268) 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:829: done (3 left) 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:830: done (2 left) 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:831: done (1 left) 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:833: read oid 21 snap -1 2024-09-14T03:40:06.498 INFO:tasks.rados.rados.0.smithi110.stdout:833: expect deleted 2024-09-14T03:40:06.499 INFO:tasks.rados.rados.0.smithi110.stdout:834: snap_remove snap 74 2024-09-14T03:40:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[35374]: pgmap v9: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[35374]: mgrmap e34: y(active, since 4s) 2024-09-14T03:40:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[35374]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:40:06.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[27365]: pgmap v9: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:06.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[27365]: mgrmap e34: y(active, since 4s) 2024-09-14T03:40:06.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:06.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:06.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:06 smithi110 ceph-mon[27365]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:40:06.950 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:40:06] ENGINE Bus STARTING 2024-09-14T03:40:06.950 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: CherryPy Checker: 2024-09-14T03:40:06.950 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: The Application mounted at '' has an empty config. 2024-09-14T03:40:06.950 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: 2024-09-14T03:40:06.951 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:40:06] ENGINE Serving on http://:::9283 2024-09-14T03:40:06.951 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[63995]: [14/Sep/2024:03:40:06] ENGINE Bus STARTED 2024-09-14T03:40:07.208 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:07.468 INFO:tasks.rados.rados.0.smithi110.stdout:834: done (2 left) 2024-09-14T03:40:07.468 INFO:tasks.rados.rados.0.smithi110.stdout:835: rollback oid 19 current snap is 81 2024-09-14T03:40:07.468 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 76 2024-09-14T03:40:07.468 INFO:tasks.rados.rados.0.smithi110.stdout:836: read oid 6 snap -1 2024-09-14T03:40:07.468 INFO:tasks.rados.rados.0.smithi110.stdout:836: expect (ObjNum 244 snap 69 seq_num 244) 2024-09-14T03:40:07.468 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 81 2024-09-14T03:40:07.468 INFO:tasks.rados.rados.0.smithi110.stdout:837: seq_num 269 ranges {2097152=507904,2605056=630784,3235840=557056,3792896=417792} 2024-09-14T03:40:07.483 INFO:tasks.rados.rados.0.smithi110.stdout:837: writing smithi11075562-10 from 2097152 to 2605056 tid 1 2024-09-14T03:40:07.487 INFO:tasks.rados.rados.0.smithi110.stdout:837: writing smithi11075562-10 from 2605056 to 3235840 tid 2 2024-09-14T03:40:07.490 INFO:tasks.rados.rados.0.smithi110.stdout:837: writing smithi11075562-10 from 3235840 to 3792896 tid 3 2024-09-14T03:40:07.493 INFO:tasks.rados.rados.0.smithi110.stdout:837: writing smithi11075562-10 from 3792896 to 4210688 tid 4 2024-09-14T03:40:07.493 INFO:tasks.rados.rados.0.smithi110.stdout:838: setattr oid 17 current snap is 81 2024-09-14T03:40:07.493 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 96 (ObjNum 129 snap 0 seq_num 65177920) dirty exists 2024-09-14T03:40:07.493 INFO:tasks.rados.rados.0.smithi110.stdout:835: finishing rollback tid 0 to smithi11075562-19 2024-09-14T03:40:07.493 INFO:tasks.rados.rados.0.smithi110.stdout:835: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:40:07.493 INFO:tasks.rados.rados.0.smithi110.stdout:835: finishing rollback tid 2 to smithi11075562-19 2024-09-14T03:40:07.494 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 369 (ObjNum 235 snap 69 seq_num 235) dirty exists 2024-09-14T03:40:07.494 INFO:tasks.rados.rados.0.smithi110.stdout:837: finishing write tid 1 to smithi11075562-10 2024-09-14T03:40:07.494 INFO:tasks.rados.rados.0.smithi110.stdout:837: finishing write tid 2 to smithi11075562-10 2024-09-14T03:40:07.494 INFO:tasks.rados.rados.0.smithi110.stdout:837: finishing write tid 3 to smithi11075562-10 2024-09-14T03:40:07.495 INFO:tasks.rados.rados.0.smithi110.stdout:832: done (5 left) 2024-09-14T03:40:07.495 INFO:tasks.rados.rados.0.smithi110.stdout:833: done (4 left) 2024-09-14T03:40:07.495 INFO:tasks.rados.rados.0.smithi110.stdout:835: done (3 left) 2024-09-14T03:40:07.495 INFO:tasks.rados.rados.0.smithi110.stdout:839: rmattr oid 4 current snap is 81 2024-09-14T03:40:07.495 INFO:tasks.rados.rados.0.smithi110.stdout:840: setattr oid 26 current snap is 81 2024-09-14T03:40:07.496 INFO:tasks.rados.rados.0.smithi110.stdout:837: finishing write tid 4 to smithi11075562-10 2024-09-14T03:40:07.496 INFO:tasks.rados.rados.0.smithi110.stdout:837: finishing write tid 5 to smithi11075562-10 2024-09-14T03:40:07.496 INFO:tasks.rados.rados.0.smithi110.stdout:837: finishing write tid 6 to smithi11075562-10 2024-09-14T03:40:07.500 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 251 (ObjNum 269 snap 81 seq_num 269) dirty exists 2024-09-14T03:40:07.500 INFO:tasks.rados.rados.0.smithi110.stdout:837: left oid 10 (ObjNum 269 snap 81 seq_num 269) 2024-09-14T03:40:07.501 INFO:tasks.rados.rados.0.smithi110.stdout:837: done (4 left) 2024-09-14T03:40:07.501 INFO:tasks.rados.rados.0.smithi110.stdout:841: delete oid 35 current snap is 81 2024-09-14T03:40:07.501 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 382 (ObjNum 897 snap 0 seq_num 64680560) dirty exists 2024-09-14T03:40:07.502 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 304 (ObjNum 255 snap 76 seq_num 255) dirty exists 2024-09-14T03:40:07.502 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 135 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:40:07.503 INFO:tasks.rados.rados.0.smithi110.stdout:836: expect (ObjNum 244 snap 69 seq_num 244) 2024-09-14T03:40:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: Standby manager daemon x started 2024-09-14T03:40:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: Standby manager daemon x started 2024-09-14T03:40:07.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:07.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:07.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:07.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:07.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:07 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: Standby manager daemon x started 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2364322557' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:07 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:836: done (4 left) 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:838: done (3 left) 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:839: done (2 left) 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:840: done (1 left) 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:841: done (0 left) 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:842: copy_from oid 8 from oid 47 current snap is 81 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:843: read oid 25 snap -1 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:843: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:40:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:844: setattr oid 13 current snap is 81 2024-09-14T03:40:07.802 INFO:tasks.rados.rados.0.smithi110.stdout:845: rollback oid 14 current snap is 81 2024-09-14T03:40:07.802 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 14 to 67 2024-09-14T03:40:07.802 INFO:tasks.rados.rados.0.smithi110.stdout:846: copy_from oid 46 from oid 20 current snap is 81 2024-09-14T03:40:07.802 INFO:tasks.rados.rados.0.smithi110.stdout:847: read oid 37 snap -1 2024-09-14T03:40:07.802 INFO:tasks.rados.rados.0.smithi110.stdout:847: expect deleted 2024-09-14T03:40:07.803 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 81 2024-09-14T03:40:07.803 INFO:tasks.rados.rados.0.smithi110.stdout:848: seq_num 271 ranges {0=540672,540672=802816,1343488=516096,1859584=286720} 2024-09-14T03:40:07.805 INFO:tasks.rados.rados.0.smithi110.stdout:848: writing smithi11075562-9 from 0 to 540672 tid 1 2024-09-14T03:40:07.809 INFO:tasks.rados.rados.0.smithi110.stdout:848: writing smithi11075562-9 from 540672 to 1343488 tid 2 2024-09-14T03:40:07.813 INFO:tasks.rados.rados.0.smithi110.stdout:848: writing smithi11075562-9 from 1343488 to 1859584 tid 3 2024-09-14T03:40:07.815 INFO:tasks.rados.rados.0.smithi110.stdout:848: writing smithi11075562-9 from 1859584 to 2146304 tid 4 2024-09-14T03:40:07.815 INFO:tasks.rados.rados.0.smithi110.stdout:846: finishing copy_from to smithi11075562-46 2024-09-14T03:40:07.815 INFO:tasks.rados.rados.0.smithi110.stdout:846: got expected ENOENT (src dne) 2024-09-14T03:40:07.815 INFO:tasks.rados.rados.0.smithi110.stdout:846: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:40:07.815 INFO:tasks.rados.rados.0.smithi110.stdout:845: finishing rollback tid 0 to smithi11075562-14 2024-09-14T03:40:07.815 INFO:tasks.rados.rados.0.smithi110.stdout:845: finishing rollback tid 1 to smithi11075562-14 2024-09-14T03:40:07.816 INFO:tasks.rados.rados.0.smithi110.stdout:845: finishing rollback tid 2 to smithi11075562-14 2024-09-14T03:40:07.816 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 441 (ObjNum 184 snap 51 seq_num 184) dirty exists 2024-09-14T03:40:07.816 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 472 (ObjNum 254 snap 76 seq_num 254) dirty exists 2024-09-14T03:40:07.816 INFO:tasks.rados.rados.0.smithi110.stdout:848: finishing write tid 1 to smithi11075562-9 2024-09-14T03:40:07.816 INFO:tasks.rados.rados.0.smithi110.stdout:843: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:848: finishing write tid 2 to smithi11075562-9 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:842: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:843: done (6 left) 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:844: done (5 left) 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:845: done (4 left) 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:846: done (3 left) 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:847: done (2 left) 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:849: read oid 17 snap -1 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:849: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:40:07.877 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 81 2024-09-14T03:40:07.878 INFO:tasks.rados.rados.0.smithi110.stdout:850: seq_num 272 ranges {1245184=565248,1810432=688128,2498560=729088,3227648=188416} 2024-09-14T03:40:07.887 INFO:tasks.rados.rados.0.smithi110.stdout:850: writing smithi11075562-22 from 1245184 to 1810432 tid 1 2024-09-14T03:40:07.891 INFO:tasks.rados.rados.0.smithi110.stdout:850: writing smithi11075562-22 from 1810432 to 2498560 tid 2 2024-09-14T03:40:07.895 INFO:tasks.rados.rados.0.smithi110.stdout:850: writing smithi11075562-22 from 2498560 to 3227648 tid 3 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:850: writing smithi11075562-22 from 3227648 to 3416064 tid 4 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:851: read oid 12 snap -1 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:851: expect deleted 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:852: rmattr oid 38 current snap is 81 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:842: finishing copy_from to smithi11075562-8 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 472 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:848: finishing write tid 3 to smithi11075562-9 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:848: finishing write tid 4 to smithi11075562-9 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:848: finishing write tid 5 to smithi11075562-9 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:842: done (5 left) 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:853: copy_from oid 33 from oid 47 current snap is 81 2024-09-14T03:40:07.897 INFO:tasks.rados.rados.0.smithi110.stdout:848: finishing write tid 6 to smithi11075562-9 2024-09-14T03:40:07.898 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 309 (ObjNum 271 snap 81 seq_num 271) dirty exists 2024-09-14T03:40:07.898 INFO:tasks.rados.rados.0.smithi110.stdout:848: left oid 9 (ObjNum 271 snap 81 seq_num 271) 2024-09-14T03:40:07.898 INFO:tasks.rados.rados.0.smithi110.stdout:849: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:40:07.901 INFO:tasks.rados.rados.0.smithi110.stdout:850: finishing write tid 1 to smithi11075562-22 2024-09-14T03:40:07.901 INFO:tasks.rados.rados.0.smithi110.stdout:850: finishing write tid 2 to smithi11075562-22 2024-09-14T03:40:07.901 INFO:tasks.rados.rados.0.smithi110.stdout:848: done (5 left) 2024-09-14T03:40:07.901 INFO:tasks.rados.rados.0.smithi110.stdout:849: done (4 left) 2024-09-14T03:40:07.901 INFO:tasks.rados.rados.0.smithi110.stdout:854: rmattr oid 5 current snap is 81 2024-09-14T03:40:07.901 INFO:tasks.rados.rados.0.smithi110.stdout:854: done (4 left) 2024-09-14T03:40:07.901 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 81 2024-09-14T03:40:07.901 INFO:tasks.rados.rados.0.smithi110.stdout:855: seq_num 273 ranges {0=581632,581632=565248,1146880=532480,1679360=507904} 2024-09-14T03:40:07.904 INFO:tasks.rados.rados.0.smithi110.stdout:855: writing smithi11075562-21 from 0 to 581632 tid 1 2024-09-14T03:40:07.907 INFO:tasks.rados.rados.0.smithi110.stdout:855: writing smithi11075562-21 from 581632 to 1146880 tid 2 2024-09-14T03:40:07.910 INFO:tasks.rados.rados.0.smithi110.stdout:855: writing smithi11075562-21 from 1146880 to 1679360 tid 3 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:855: writing smithi11075562-21 from 1679360 to 2187264 tid 4 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:856: delete oid 41 current snap is 81 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:850: finishing write tid 3 to smithi11075562-22 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 218 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:850: finishing write tid 4 to smithi11075562-22 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:850: finishing write tid 5 to smithi11075562-22 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:850: finishing write tid 6 to smithi11075562-22 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 477 (ObjNum 272 snap 81 seq_num 272) dirty exists 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:850: left oid 22 (ObjNum 272 snap 81 seq_num 272) 2024-09-14T03:40:07.914 INFO:tasks.rados.rados.0.smithi110.stdout:853: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:40:07.915 INFO:tasks.rados.rados.0.smithi110.stdout:853: finishing copy_from to smithi11075562-33 2024-09-14T03:40:07.915 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 472 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:40:07.915 INFO:tasks.rados.rados.0.smithi110.stdout:855: finishing write tid 1 to smithi11075562-21 2024-09-14T03:40:07.915 INFO:tasks.rados.rados.0.smithi110.stdout:855: finishing write tid 2 to smithi11075562-21 2024-09-14T03:40:07.915 INFO:tasks.rados.rados.0.smithi110.stdout:855: finishing write tid 3 to smithi11075562-21 2024-09-14T03:40:07.916 INFO:tasks.rados.rados.0.smithi110.stdout:855: finishing write tid 4 to smithi11075562-21 2024-09-14T03:40:07.916 INFO:tasks.rados.rados.0.smithi110.stdout:855: finishing write tid 5 to smithi11075562-21 2024-09-14T03:40:07.917 INFO:tasks.rados.rados.0.smithi110.stdout:855: finishing write tid 6 to smithi11075562-21 2024-09-14T03:40:07.917 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 446 (ObjNum 273 snap 81 seq_num 273) dirty exists 2024-09-14T03:40:07.917 INFO:tasks.rados.rados.0.smithi110.stdout:855: left oid 21 (ObjNum 273 snap 81 seq_num 273) 2024-09-14T03:40:07.918 INFO:tasks.rados.rados.0.smithi110.stdout:850: done (5 left) 2024-09-14T03:40:07.918 INFO:tasks.rados.rados.0.smithi110.stdout:851: done (4 left) 2024-09-14T03:40:07.918 INFO:tasks.rados.rados.0.smithi110.stdout:852: done (3 left) 2024-09-14T03:40:07.918 INFO:tasks.rados.rados.0.smithi110.stdout:853: done (2 left) 2024-09-14T03:40:07.918 INFO:tasks.rados.rados.0.smithi110.stdout:855: done (1 left) 2024-09-14T03:40:07.918 INFO:tasks.rados.rados.0.smithi110.stdout:856: done (0 left) 2024-09-14T03:40:07.918 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 81 2024-09-14T03:40:07.918 INFO:tasks.rados.rados.0.smithi110.stdout:857: seq_num 275 ranges {1318912=614400,1933312=786432,2719744=499712,3219456=319488} 2024-09-14T03:40:07.928 INFO:tasks.rados.rados.0.smithi110.stdout:857: writing smithi11075562-14 from 1318912 to 1933312 tid 1 2024-09-14T03:40:07.932 INFO:tasks.rados.rados.0.smithi110.stdout:857: writing smithi11075562-14 from 1933312 to 2719744 tid 2 2024-09-14T03:40:07.935 INFO:tasks.rados.rados.0.smithi110.stdout:857: writing smithi11075562-14 from 2719744 to 3219456 tid 3 2024-09-14T03:40:07.938 INFO:tasks.rados.rados.0.smithi110.stdout:857: writing smithi11075562-14 from 3219456 to 3538944 tid 4 2024-09-14T03:40:07.938 INFO:tasks.rados.rados.0.smithi110.stdout:858: read oid 31 snap -1 2024-09-14T03:40:07.938 INFO:tasks.rados.rados.0.smithi110.stdout:858: expect deleted 2024-09-14T03:40:07.938 INFO:tasks.rados.rados.0.smithi110.stdout:859: rollback oid 17 current snap is 81 2024-09-14T03:40:07.938 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 65 2024-09-14T03:40:07.938 INFO:tasks.rados.rados.0.smithi110.stdout:860: rollback oid 2 current snap is 81 2024-09-14T03:40:07.939 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 2 to 78 2024-09-14T03:40:07.939 INFO:tasks.rados.rados.0.smithi110.stdout:857: finishing write tid 1 to smithi11075562-14 2024-09-14T03:40:07.939 INFO:tasks.rados.rados.0.smithi110.stdout:857: finishing write tid 2 to smithi11075562-14 2024-09-14T03:40:07.939 INFO:tasks.rados.rados.0.smithi110.stdout:857: finishing write tid 3 to smithi11075562-14 2024-09-14T03:40:07.939 INFO:tasks.rados.rados.0.smithi110.stdout:861: read oid 39 snap -1 2024-09-14T03:40:07.939 INFO:tasks.rados.rados.0.smithi110.stdout:861: expect (ObjNum 260 snap 79 seq_num 260) 2024-09-14T03:40:07.939 INFO:tasks.rados.rados.0.smithi110.stdout:862: snap_create 2024-09-14T03:40:07.940 INFO:tasks.rados.rados.0.smithi110.stdout:860: finishing rollback tid 0 to smithi11075562-2 2024-09-14T03:40:07.940 INFO:tasks.rados.rados.0.smithi110.stdout:860: finishing rollback tid 1 to smithi11075562-2 2024-09-14T03:40:07.941 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 297 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-14T03:40:07.941 INFO:tasks.rados.rados.0.smithi110.stdout:859: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:40:07.941 INFO:tasks.rados.rados.0.smithi110.stdout:857: finishing write tid 4 to smithi11075562-14 2024-09-14T03:40:07.941 INFO:tasks.rados.rados.0.smithi110.stdout:859: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:40:07.941 INFO:tasks.rados.rados.0.smithi110.stdout:859: finishing rollback tid 2 to smithi11075562-17 2024-09-14T03:40:07.941 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 385 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-14T03:40:07.941 INFO:tasks.rados.rados.0.smithi110.stdout:857: finishing write tid 5 to smithi11075562-14 2024-09-14T03:40:07.954 INFO:tasks.rados.rados.0.smithi110.stdout:857: finishing write tid 6 to smithi11075562-14 2024-09-14T03:40:07.954 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 453 (ObjNum 275 snap 81 seq_num 275) dirty exists 2024-09-14T03:40:07.954 INFO:tasks.rados.rados.0.smithi110.stdout:857: left oid 14 (ObjNum 275 snap 81 seq_num 275) 2024-09-14T03:40:07.958 INFO:tasks.rados.rados.0.smithi110.stdout:861: expect (ObjNum 260 snap 79 seq_num 260) 2024-09-14T03:40:08.473 INFO:tasks.rados.rados.0.smithi110.stdout:857: done (5 left) 2024-09-14T03:40:08.473 INFO:tasks.rados.rados.0.smithi110.stdout:858: done (4 left) 2024-09-14T03:40:08.473 INFO:tasks.rados.rados.0.smithi110.stdout:859: done (3 left) 2024-09-14T03:40:08.473 INFO:tasks.rados.rados.0.smithi110.stdout:860: done (2 left) 2024-09-14T03:40:08.474 INFO:tasks.rados.rados.0.smithi110.stdout:861: done (1 left) 2024-09-14T03:40:08.474 INFO:tasks.rados.rados.0.smithi110.stdout:862: done (0 left) 2024-09-14T03:40:08.474 INFO:tasks.rados.rados.0.smithi110.stdout:863: rollback oid 34 current snap is 82 2024-09-14T03:40:08.474 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 64 2024-09-14T03:40:08.474 INFO:tasks.rados.rados.0.smithi110.stdout:864: delete oid 12 current snap is 82 2024-09-14T03:40:08.477 INFO:tasks.rados.rados.0.smithi110.stdout:864: done (1 left) 2024-09-14T03:40:08.477 INFO:tasks.rados.rados.0.smithi110.stdout:865: rmattr oid 37 current snap is 82 2024-09-14T03:40:08.477 INFO:tasks.rados.rados.0.smithi110.stdout:865: done (1 left) 2024-09-14T03:40:08.477 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 82 2024-09-14T03:40:08.477 INFO:tasks.rados.rados.0.smithi110.stdout:866: seq_num 277 ranges {0=647168,647168=557056,1204224=516096,1720320=532480} 2024-09-14T03:40:08.480 INFO:tasks.rados.rados.0.smithi110.stdout:866: writing smithi11075562-2 from 0 to 647168 tid 1 2024-09-14T03:40:08.484 INFO:tasks.rados.rados.0.smithi110.stdout:866: writing smithi11075562-2 from 647168 to 1204224 tid 2 2024-09-14T03:40:08.487 INFO:tasks.rados.rados.0.smithi110.stdout:866: writing smithi11075562-2 from 1204224 to 1720320 tid 3 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:866: writing smithi11075562-2 from 1720320 to 2252800 tid 4 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:867: rollback oid 28 current snap is 82 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 28 to 80 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:863: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:868: read oid 43 snap -1 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:868: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:869: snap_create 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:863: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:863: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 307 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:866: finishing write tid 1 to smithi11075562-2 2024-09-14T03:40:08.491 INFO:tasks.rados.rados.0.smithi110.stdout:866: finishing write tid 2 to smithi11075562-2 2024-09-14T03:40:08.492 INFO:tasks.rados.rados.0.smithi110.stdout:866: finishing write tid 3 to smithi11075562-2 2024-09-14T03:40:08.495 INFO:tasks.rados.rados.0.smithi110.stdout:866: finishing write tid 4 to smithi11075562-2 2024-09-14T03:40:08.495 INFO:tasks.rados.rados.0.smithi110.stdout:866: finishing write tid 5 to smithi11075562-2 2024-09-14T03:40:08.495 INFO:tasks.rados.rados.0.smithi110.stdout:867: finishing rollback tid 0 to smithi11075562-28 2024-09-14T03:40:08.495 INFO:tasks.rados.rados.0.smithi110.stdout:867: finishing rollback tid 1 to smithi11075562-28 2024-09-14T03:40:08.498 INFO:tasks.rados.rados.0.smithi110.stdout:866: finishing write tid 6 to smithi11075562-2 2024-09-14T03:40:08.498 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 302 (ObjNum 277 snap 82 seq_num 277) dirty exists 2024-09-14T03:40:08.498 INFO:tasks.rados.rados.0.smithi110.stdout:866: left oid 2 (ObjNum 277 snap 82 seq_num 277) 2024-09-14T03:40:08.498 INFO:tasks.rados.rados.0.smithi110.stdout:867: finishing rollback tid 2 to smithi11075562-28 2024-09-14T03:40:08.498 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 306 (ObjNum 225 snap 67 seq_num 225) dirty exists 2024-09-14T03:40:08.504 INFO:tasks.rados.rados.0.smithi110.stdout:868: expect (ObjNum 208 snap 62 seq_num 208) 2024-09-14T03:40:08.807 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:08 smithi110 ceph-mon[27365]: pgmap v11: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:08.807 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:08 smithi110 ceph-mon[27365]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:40:08.807 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:08 smithi110 ceph-mon[27365]: mgrmap e35: y(active, since 6s), standbys: x 2024-09-14T03:40:08.807 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:08 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:08.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:08 smithi110 ceph-mon[35374]: pgmap v11: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:08.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:08 smithi110 ceph-mon[35374]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:40:08.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:08 smithi110 ceph-mon[35374]: mgrmap e35: y(active, since 6s), standbys: x 2024-09-14T03:40:08.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:08 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:08.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:08 smithi195 ceph-mon[30695]: pgmap v11: 121 pgs: 121 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:08.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:08 smithi195 ceph-mon[30695]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:40:08.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:08 smithi195 ceph-mon[30695]: mgrmap e35: y(active, since 6s), standbys: x 2024-09-14T03:40:08.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:08 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:09.478 INFO:tasks.rados.rados.0.smithi110.stdout:863: done (4 left) 2024-09-14T03:40:09.478 INFO:tasks.rados.rados.0.smithi110.stdout:866: done (3 left) 2024-09-14T03:40:09.478 INFO:tasks.rados.rados.0.smithi110.stdout:867: done (2 left) 2024-09-14T03:40:09.478 INFO:tasks.rados.rados.0.smithi110.stdout:868: done (1 left) 2024-09-14T03:40:09.479 INFO:tasks.rados.rados.0.smithi110.stdout:869: done (0 left) 2024-09-14T03:40:09.479 INFO:tasks.rados.rados.0.smithi110.stdout:870: snap_create 2024-09-14T03:40:09.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:09 smithi110 ceph-mon[27365]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:40:09.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:09 smithi110 ceph-mon[35374]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:40:09.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:09 smithi195 ceph-mon[30695]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:40:10.045 INFO:tasks.rados.rados.0.smithi110.stdout:870: done (0 left) 2024-09-14T03:40:10.045 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 84 2024-09-14T03:40:10.045 INFO:tasks.rados.rados.0.smithi110.stdout:871: seq_num 278 ranges {3375104=663552,4038656=442368,4481024=720896,5201920=442368} 2024-09-14T03:40:10.068 INFO:tasks.rados.rados.0.smithi110.stdout:871: writing smithi11075562-18 from 3375104 to 4038656 tid 1 2024-09-14T03:40:10.072 INFO:tasks.rados.rados.0.smithi110.stdout:871: writing smithi11075562-18 from 4038656 to 4481024 tid 2 2024-09-14T03:40:10.078 INFO:tasks.rados.rados.0.smithi110.stdout:871: writing smithi11075562-18 from 4481024 to 5201920 tid 3 2024-09-14T03:40:10.081 INFO:tasks.rados.rados.0.smithi110.stdout:871: writing smithi11075562-18 from 5201920 to 5644288 tid 4 2024-09-14T03:40:10.081 INFO:tasks.rados.rados.0.smithi110.stdout:872: copy_from oid 5 from oid 33 current snap is 84 2024-09-14T03:40:10.081 INFO:tasks.rados.rados.0.smithi110.stdout:871: finishing write tid 1 to smithi11075562-18 2024-09-14T03:40:10.081 INFO:tasks.rados.rados.0.smithi110.stdout:871: finishing write tid 2 to smithi11075562-18 2024-09-14T03:40:10.082 INFO:tasks.rados.rados.0.smithi110.stdout:871: finishing write tid 3 to smithi11075562-18 2024-09-14T03:40:10.082 INFO:tasks.rados.rados.0.smithi110.stdout:873: snap_remove snap 76 2024-09-14T03:40:10.475 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:10.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: pgmap v14: 121 pgs: 121 active+clean; 235 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:10.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[27365]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: pgmap v14: 121 pgs: 121 active+clean; 235 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:40:10.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.699 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:40:10.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:10 smithi110 ceph-mon[35374]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:40:10.741 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: pgmap v14: 121 pgs: 121 active+clean; 235 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-09-14T03:40:10.741 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:40:10.741 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.741 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.741 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:10.741 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:10.741 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:40:10.742 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:10 smithi195 ceph-mon[30695]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:40:11.049 INFO:tasks.rados.rados.0.smithi110.stdout:871: finishing write tid 4 to smithi11075562-18 2024-09-14T03:40:11.049 INFO:tasks.rados.rados.0.smithi110.stdout:871: finishing write tid 5 to smithi11075562-18 2024-09-14T03:40:11.049 INFO:tasks.rados.rados.0.smithi110.stdout:871: finishing write tid 6 to smithi11075562-18 2024-09-14T03:40:11.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 375 (ObjNum 278 snap 84 seq_num 278) dirty exists 2024-09-14T03:40:11.049 INFO:tasks.rados.rados.0.smithi110.stdout:871: left oid 18 (ObjNum 278 snap 84 seq_num 278) 2024-09-14T03:40:11.049 INFO:tasks.rados.rados.0.smithi110.stdout:872: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:40:11.049 INFO:tasks.rados.rados.0.smithi110.stdout:872: finishing copy_from to smithi11075562-5 2024-09-14T03:40:11.050 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 472 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:40:11.050 INFO:tasks.rados.rados.0.smithi110.stdout:871: done (2 left) 2024-09-14T03:40:11.050 INFO:tasks.rados.rados.0.smithi110.stdout:872: done (1 left) 2024-09-14T03:40:11.050 INFO:tasks.rados.rados.0.smithi110.stdout:873: done (0 left) 2024-09-14T03:40:11.050 INFO:tasks.rados.rados.0.smithi110.stdout:874: rollback oid 28 current snap is 84 2024-09-14T03:40:11.050 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 28 to 80 2024-09-14T03:40:11.050 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 84 2024-09-14T03:40:11.050 INFO:tasks.rados.rados.0.smithi110.stdout:875: seq_num 279 ranges {2473984=679936,3153920=720896,3874816=483328,4358144=401408} 2024-09-14T03:40:11.067 INFO:tasks.rados.rados.0.smithi110.stdout:875: writing smithi11075562-42 from 2473984 to 3153920 tid 1 2024-09-14T03:40:11.070 INFO:tasks.rados.rados.0.smithi110.stdout:875: writing smithi11075562-42 from 3153920 to 3874816 tid 2 2024-09-14T03:40:11.073 INFO:tasks.rados.rados.0.smithi110.stdout:875: writing smithi11075562-42 from 3874816 to 4358144 tid 3 2024-09-14T03:40:11.076 INFO:tasks.rados.rados.0.smithi110.stdout:875: writing smithi11075562-42 from 4358144 to 4759552 tid 4 2024-09-14T03:40:11.076 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 84 2024-09-14T03:40:11.076 INFO:tasks.rados.rados.0.smithi110.stdout:874: finishing rollback tid 0 to smithi11075562-28 2024-09-14T03:40:11.076 INFO:tasks.rados.rados.0.smithi110.stdout:876: seq_num 280 ranges {1851392=696320,2547712=606208,3153920=688128,3842048=311296} 2024-09-14T03:40:11.089 INFO:tasks.rados.rados.0.smithi110.stdout:876: writing smithi11075562-23 from 1851392 to 2547712 tid 1 2024-09-14T03:40:11.093 INFO:tasks.rados.rados.0.smithi110.stdout:876: writing smithi11075562-23 from 2547712 to 3153920 tid 2 2024-09-14T03:40:11.097 INFO:tasks.rados.rados.0.smithi110.stdout:876: writing smithi11075562-23 from 3153920 to 3842048 tid 3 2024-09-14T03:40:11.099 INFO:tasks.rados.rados.0.smithi110.stdout:876: writing smithi11075562-23 from 3842048 to 4153344 tid 4 2024-09-14T03:40:11.099 INFO:tasks.rados.rados.0.smithi110.stdout:877: snap_create 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:874: finishing rollback tid 1 to smithi11075562-28 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:874: finishing rollback tid 2 to smithi11075562-28 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 312 (ObjNum 225 snap 67 seq_num 225) dirty exists 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:875: finishing write tid 1 to smithi11075562-42 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:875: finishing write tid 2 to smithi11075562-42 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:875: finishing write tid 3 to smithi11075562-42 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:875: finishing write tid 4 to smithi11075562-42 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:875: finishing write tid 5 to smithi11075562-42 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:875: finishing write tid 6 to smithi11075562-42 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 482 (ObjNum 279 snap 84 seq_num 279) dirty exists 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:875: left oid 42 (ObjNum 279 snap 84 seq_num 279) 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:876: finishing write tid 1 to smithi11075562-23 2024-09-14T03:40:11.100 INFO:tasks.rados.rados.0.smithi110.stdout:876: finishing write tid 2 to smithi11075562-23 2024-09-14T03:40:11.101 INFO:tasks.rados.rados.0.smithi110.stdout:876: finishing write tid 3 to smithi11075562-23 2024-09-14T03:40:11.101 INFO:tasks.rados.rados.0.smithi110.stdout:876: finishing write tid 4 to smithi11075562-23 2024-09-14T03:40:11.101 INFO:tasks.rados.rados.0.smithi110.stdout:876: finishing write tid 5 to smithi11075562-23 2024-09-14T03:40:11.101 INFO:tasks.rados.rados.0.smithi110.stdout:876: finishing write tid 6 to smithi11075562-23 2024-09-14T03:40:11.101 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 256 (ObjNum 280 snap 84 seq_num 280) dirty exists 2024-09-14T03:40:11.102 INFO:tasks.rados.rados.0.smithi110.stdout:876: left oid 23 (ObjNum 280 snap 84 seq_num 280) 2024-09-14T03:40:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:874: done (3 left) 2024-09-14T03:40:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:875: done (2 left) 2024-09-14T03:40:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:876: done (1 left) 2024-09-14T03:40:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:877: done (0 left) 2024-09-14T03:40:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 85 2024-09-14T03:40:12.053 INFO:tasks.rados.rados.0.smithi110.stdout:878: seq_num 281 ranges {2269184=712704,2981888=491520,3473408=491520,3964928=622592} 2024-09-14T03:40:12.069 INFO:tasks.rados.rados.0.smithi110.stdout:878: writing smithi11075562-45 from 2269184 to 2981888 tid 1 2024-09-14T03:40:12.072 INFO:tasks.rados.rados.0.smithi110.stdout:878: writing smithi11075562-45 from 2981888 to 3473408 tid 2 2024-09-14T03:40:12.075 INFO:tasks.rados.rados.0.smithi110.stdout:878: writing smithi11075562-45 from 3473408 to 3964928 tid 3 2024-09-14T03:40:12.078 INFO:tasks.rados.rados.0.smithi110.stdout:878: writing smithi11075562-45 from 3964928 to 4587520 tid 4 2024-09-14T03:40:12.079 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 85 2024-09-14T03:40:12.079 INFO:tasks.rados.rados.0.smithi110.stdout:878: finishing write tid 1 to smithi11075562-45 2024-09-14T03:40:12.079 INFO:tasks.rados.rados.0.smithi110.stdout:878: finishing write tid 2 to smithi11075562-45 2024-09-14T03:40:12.079 INFO:tasks.rados.rados.0.smithi110.stdout:878: finishing write tid 3 to smithi11075562-45 2024-09-14T03:40:12.079 INFO:tasks.rados.rados.0.smithi110.stdout:879: seq_num 282 ranges {4210688=729088,4939776=425984,5365760=704512,6070272=475136} 2024-09-14T03:40:12.106 INFO:tasks.rados.rados.0.smithi110.stdout:879: writing smithi11075562-10 from 4210688 to 4939776 tid 1 2024-09-14T03:40:12.108 INFO:tasks.rados.rados.0.smithi110.stdout:879: writing smithi11075562-10 from 4939776 to 5365760 tid 2 2024-09-14T03:40:12.112 INFO:tasks.rados.rados.0.smithi110.stdout:879: writing smithi11075562-10 from 5365760 to 6070272 tid 3 2024-09-14T03:40:12.116 INFO:tasks.rados.rados.0.smithi110.stdout:879: writing smithi11075562-10 from 6070272 to 6545408 tid 4 2024-09-14T03:40:12.116 INFO:tasks.rados.rados.0.smithi110.stdout:880: delete oid 7 current snap is 85 2024-09-14T03:40:12.116 INFO:tasks.rados.rados.0.smithi110.stdout:878: finishing write tid 4 to smithi11075562-45 2024-09-14T03:40:12.117 INFO:tasks.rados.rados.0.smithi110.stdout:878: finishing write tid 5 to smithi11075562-45 2024-09-14T03:40:12.117 INFO:tasks.rados.rados.0.smithi110.stdout:878: finishing write tid 6 to smithi11075562-45 2024-09-14T03:40:12.117 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 517 (ObjNum 281 snap 85 seq_num 281) dirty exists 2024-09-14T03:40:12.117 INFO:tasks.rados.rados.0.smithi110.stdout:878: left oid 45 (ObjNum 281 snap 85 seq_num 281) 2024-09-14T03:40:12.117 INFO:tasks.rados.rados.0.smithi110.stdout:879: finishing write tid 1 to smithi11075562-10 2024-09-14T03:40:12.117 INFO:tasks.rados.rados.0.smithi110.stdout:879: finishing write tid 2 to smithi11075562-10 2024-09-14T03:40:12.117 INFO:tasks.rados.rados.0.smithi110.stdout:879: finishing write tid 3 to smithi11075562-10 2024-09-14T03:40:12.118 INFO:tasks.rados.rados.0.smithi110.stdout:878: done (2 left) 2024-09-14T03:40:12.118 INFO:tasks.rados.rados.0.smithi110.stdout:880: done (1 left) 2024-09-14T03:40:12.118 INFO:tasks.rados.rados.0.smithi110.stdout:881: rmattr oid 20 current snap is 85 2024-09-14T03:40:12.118 INFO:tasks.rados.rados.0.smithi110.stdout:881: done (1 left) 2024-09-14T03:40:12.118 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 85 2024-09-14T03:40:12.118 INFO:tasks.rados.rados.0.smithi110.stdout:882: seq_num 284 ranges {1302528=761856,2064384=598016,2662400=671744,3334144=335872} 2024-09-14T03:40:12.128 INFO:tasks.rados.rados.0.smithi110.stdout:882: writing smithi11075562-24 from 1302528 to 2064384 tid 1 2024-09-14T03:40:12.132 INFO:tasks.rados.rados.0.smithi110.stdout:882: writing smithi11075562-24 from 2064384 to 2662400 tid 2 2024-09-14T03:40:12.136 INFO:tasks.rados.rados.0.smithi110.stdout:882: writing smithi11075562-24 from 2662400 to 3334144 tid 3 2024-09-14T03:40:12.139 INFO:tasks.rados.rados.0.smithi110.stdout:882: writing smithi11075562-24 from 3334144 to 3670016 tid 4 2024-09-14T03:40:12.139 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 85 2024-09-14T03:40:12.139 INFO:tasks.rados.rados.0.smithi110.stdout:879: finishing write tid 4 to smithi11075562-10 2024-09-14T03:40:12.139 INFO:tasks.rados.rados.0.smithi110.stdout:879: finishing write tid 5 to smithi11075562-10 2024-09-14T03:40:12.139 INFO:tasks.rados.rados.0.smithi110.stdout:879: finishing write tid 6 to smithi11075562-10 2024-09-14T03:40:12.139 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 261 (ObjNum 282 snap 85 seq_num 282) dirty exists 2024-09-14T03:40:12.139 INFO:tasks.rados.rados.0.smithi110.stdout:879: left oid 10 (ObjNum 282 snap 85 seq_num 282) 2024-09-14T03:40:12.140 INFO:tasks.rados.rados.0.smithi110.stdout:882: finishing write tid 1 to smithi11075562-24 2024-09-14T03:40:12.140 INFO:tasks.rados.rados.0.smithi110.stdout:882: finishing write tid 2 to smithi11075562-24 2024-09-14T03:40:12.140 INFO:tasks.rados.rados.0.smithi110.stdout:883: seq_num 285 ranges {0=778240,778240=475136,1253376=475136,1728512=655360} 2024-09-14T03:40:12.143 INFO:tasks.rados.rados.0.smithi110.stdout:883: writing smithi11075562-37 from 0 to 778240 tid 1 2024-09-14T03:40:12.146 INFO:tasks.rados.rados.0.smithi110.stdout:883: writing smithi11075562-37 from 778240 to 1253376 tid 2 2024-09-14T03:40:12.149 INFO:tasks.rados.rados.0.smithi110.stdout:883: writing smithi11075562-37 from 1253376 to 1728512 tid 3 2024-09-14T03:40:12.153 INFO:tasks.rados.rados.0.smithi110.stdout:883: writing smithi11075562-37 from 1728512 to 2383872 tid 4 2024-09-14T03:40:12.153 INFO:tasks.rados.rados.0.smithi110.stdout:879: done (2 left) 2024-09-14T03:40:12.153 INFO:tasks.rados.rados.0.smithi110.stdout:884: delete oid 20 current snap is 85 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:882: finishing write tid 3 to smithi11075562-24 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:882: finishing write tid 4 to smithi11075562-24 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:882: finishing write tid 5 to smithi11075562-24 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:882: finishing write tid 6 to smithi11075562-24 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 266 (ObjNum 284 snap 85 seq_num 284) dirty exists 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:882: left oid 24 (ObjNum 284 snap 85 seq_num 284) 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:883: finishing write tid 1 to smithi11075562-37 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:883: finishing write tid 2 to smithi11075562-37 2024-09-14T03:40:12.154 INFO:tasks.rados.rados.0.smithi110.stdout:883: finishing write tid 3 to smithi11075562-37 2024-09-14T03:40:12.155 INFO:tasks.rados.rados.0.smithi110.stdout:882: done (2 left) 2024-09-14T03:40:12.155 INFO:tasks.rados.rados.0.smithi110.stdout:884: done (1 left) 2024-09-14T03:40:12.155 INFO:tasks.rados.rados.0.smithi110.stdout:885: snap_create 2024-09-14T03:40:12.157 INFO:tasks.rados.rados.0.smithi110.stdout:883: finishing write tid 4 to smithi11075562-37 2024-09-14T03:40:12.157 INFO:tasks.rados.rados.0.smithi110.stdout:883: finishing write tid 5 to smithi11075562-37 2024-09-14T03:40:12.157 INFO:tasks.rados.rados.0.smithi110.stdout:883: finishing write tid 6 to smithi11075562-37 2024-09-14T03:40:12.157 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 522 (ObjNum 285 snap 85 seq_num 285) dirty exists 2024-09-14T03:40:12.157 INFO:tasks.rados.rados.0.smithi110.stdout:883: left oid 37 (ObjNum 285 snap 85 seq_num 285) 2024-09-14T03:40:12.409 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:12 smithi195 ceph-mon[30695]: Reconfiguring daemon prometheus.a on smithi195 2024-09-14T03:40:12.410 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:12 smithi195 ceph-mon[30695]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:40:12.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:12 smithi110 ceph-mon[27365]: Reconfiguring daemon prometheus.a on smithi195 2024-09-14T03:40:12.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:12 smithi110 ceph-mon[27365]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:40:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:12 smithi110 ceph-mon[35374]: Reconfiguring daemon prometheus.a on smithi195 2024-09-14T03:40:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:12 smithi110 ceph-mon[35374]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:40:13.054 INFO:tasks.rados.rados.0.smithi110.stdout:883: done (1 left) 2024-09-14T03:40:13.054 INFO:tasks.rados.rados.0.smithi110.stdout:885: done (0 left) 2024-09-14T03:40:13.054 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 86 2024-09-14T03:40:13.055 INFO:tasks.rados.rados.0.smithi110.stdout:886: seq_num 287 ranges {7364608=8192} 2024-09-14T03:40:13.094 INFO:tasks.rados.rados.0.smithi110.stdout:886: writing smithi11075562-13 from 7364608 to 7372800 tid 1 2024-09-14T03:40:13.094 INFO:tasks.rados.rados.0.smithi110.stdout:887: delete oid 50 current snap is 86 2024-09-14T03:40:13.097 INFO:tasks.rados.rados.0.smithi110.stdout:886: finishing write tid 1 to smithi11075562-13 2024-09-14T03:40:13.098 INFO:tasks.rados.rados.0.smithi110.stdout:886: finishing write tid 2 to smithi11075562-13 2024-09-14T03:40:13.098 INFO:tasks.rados.rados.0.smithi110.stdout:887: done (1 left) 2024-09-14T03:40:13.098 INFO:tasks.rados.rados.0.smithi110.stdout:888: rollback oid 21 current snap is 86 2024-09-14T03:40:13.098 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 56 2024-09-14T03:40:13.098 INFO:tasks.rados.rados.0.smithi110.stdout:889: snap_remove snap 75 2024-09-14T03:40:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: pgmap v18: 121 pgs: 121 active+clean; 235 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:40:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: pgmap v18: 121 pgs: 121 active+clean; 235 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: pgmap v18: 121 pgs: 121 active+clean; 235 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:13.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:13.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:13.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:13.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:13 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:13.771 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 systemd[1]: Stopping Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:40:14.062 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.770Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:40:14.062 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.770Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:40:14.062 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.770Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.770Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.770Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.770Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.770Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.770Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.771Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.772Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.772Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[71658]: ts=2024-09-14T03:40:13.772Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-14T03:40:14.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:13 smithi195 podman[74193]: 2024-09-14 03:40:13.780916643 +0000 UTC m=+0.131414467 container died acbe21630a9adbbed840dd0a1e5da576ab71c3eadd21cdfae4b05a1b5979fbac (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:40:14.068 INFO:tasks.rados.rados.0.smithi110.stdout:886: finishing write tid 3 to smithi11075562-13 2024-09-14T03:40:14.068 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 485 (ObjNum 287 snap 86 seq_num 287) dirty exists 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:886: left oid 13 (ObjNum 287 snap 86 seq_num 287) 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:888: finishing rollback tid 0 to smithi11075562-21 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:888: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 462 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:886: done (2 left) 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:888: done (1 left) 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:889: done (0 left) 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:890: read oid 46 snap -1 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:890: expect deleted 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:891: read oid 12 snap -1 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:891: expect deleted 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:892: read oid 29 snap -1 2024-09-14T03:40:14.069 INFO:tasks.rados.rados.0.smithi110.stdout:892: expect deleted 2024-09-14T03:40:14.070 INFO:tasks.rados.rados.0.smithi110.stdout:893: snap_remove snap 65 2024-09-14T03:40:14.422 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 podman[74193]: 2024-09-14 03:40:14.203226204 +0000 UTC m=+0.553724027 container cleanup acbe21630a9adbbed840dd0a1e5da576ab71c3eadd21cdfae4b05a1b5979fbac (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:40:14.422 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 bash[74193]: ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:40:14.422 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:14 smithi195 ceph-mon[30695]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:40:14.422 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:14 smithi195 ceph-mon[30695]: mgrmap e36: y(active, since 11s), standbys: x 2024-09-14T03:40:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:14 smithi110 ceph-mon[27365]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:40:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:14 smithi110 ceph-mon[27365]: mgrmap e36: y(active, since 11s), standbys: x 2024-09-14T03:40:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:14 smithi110 ceph-mon[35374]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:40:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:14 smithi110 ceph-mon[35374]: mgrmap e36: y(active, since 11s), standbys: x 2024-09-14T03:40:14.672 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 podman[74208]: 2024-09-14 03:40:14.42114041 +0000 UTC m=+0.637867698 container remove acbe21630a9adbbed840dd0a1e5da576ab71c3eadd21cdfae4b05a1b5979fbac (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:40:14.765 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:40:14.940 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:40:14.940 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 systemd[1]: Stopped Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:40:14.940 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.066s CPU time. 2024-09-14T03:40:14.940 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 systemd[1]: Starting Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:40:14.940 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 podman[74299]: 2024-09-14 03:40:14.7884796 +0000 UTC m=+0.017008172 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-14T03:40:14.941 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:14 smithi195 podman[74299]: 2024-09-14 03:40:14.896890522 +0000 UTC m=+0.125419094 container create e0fe3bbc1c08b6e4d30a0d40d2270143f4113effa04a84fb9604d01aab12b1a7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:40:15.047 INFO:tasks.rados.rados.0.smithi110.stdout:890: done (3 left) 2024-09-14T03:40:15.047 INFO:tasks.rados.rados.0.smithi110.stdout:891: done (2 left) 2024-09-14T03:40:15.047 INFO:tasks.rados.rados.0.smithi110.stdout:892: done (1 left) 2024-09-14T03:40:15.047 INFO:tasks.rados.rados.0.smithi110.stdout:893: done (0 left) 2024-09-14T03:40:15.047 INFO:tasks.rados.rados.0.smithi110.stdout:894: setattr oid 27 current snap is 86 2024-09-14T03:40:15.049 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 86 2024-09-14T03:40:15.049 INFO:tasks.rados.rados.0.smithi110.stdout:895: seq_num 289 ranges {0=40960} 2024-09-14T03:40:15.049 INFO:tasks.rados.rados.0.smithi110.stdout:895: writing smithi11075562-12 from 0 to 40960 tid 1 2024-09-14T03:40:15.050 INFO:tasks.rados.rados.0.smithi110.stdout:896: read oid 21 snap 80 2024-09-14T03:40:15.050 INFO:tasks.rados.rados.0.smithi110.stdout:896: expect deleted 2024-09-14T03:40:15.050 INFO:tasks.rados.rados.0.smithi110.stdout:897: read oid 35 snap 77 2024-09-14T03:40:15.050 INFO:tasks.rados.rados.0.smithi110.stdout:897: expect (ObjNum 246 snap 69 seq_num 246) 2024-09-14T03:40:15.050 INFO:tasks.rados.rados.0.smithi110.stdout:898: snap_create 2024-09-14T03:40:15.054 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 97 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-14T03:40:15.066 INFO:tasks.rados.rados.0.smithi110.stdout:895: finishing write tid 1 to smithi11075562-12 2024-09-14T03:40:15.068 INFO:tasks.rados.rados.0.smithi110.stdout:895: finishing write tid 2 to smithi11075562-12 2024-09-14T03:40:15.072 INFO:tasks.rados.rados.0.smithi110.stdout:895: finishing write tid 3 to smithi11075562-12 2024-09-14T03:40:15.072 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 532 (ObjNum 289 snap 86 seq_num 289) dirty exists 2024-09-14T03:40:15.072 INFO:tasks.rados.rados.0.smithi110.stdout:895: left oid 12 (ObjNum 289 snap 86 seq_num 289) 2024-09-14T03:40:15.081 INFO:tasks.rados.rados.0.smithi110.stdout:897: expect (ObjNum 246 snap 69 seq_num 246) 2024-09-14T03:40:15.197 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 podman[74299]: 2024-09-14 03:40:15.174433842 +0000 UTC m=+0.402962417 container init e0fe3bbc1c08b6e4d30a0d40d2270143f4113effa04a84fb9604d01aab12b1a7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:40:15.197 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 podman[74299]: 2024-09-14 03:40:15.177380099 +0000 UTC m=+0.405908677 container start e0fe3bbc1c08b6e4d30a0d40d2270143f4113effa04a84fb9604d01aab12b1a7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:40:15.198 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-mon[30695]: pgmap v21: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 8.3 MiB/s wr, 62 op/s 2024-09-14T03:40:15.198 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-mon[30695]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:40:15.198 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-mon[30695]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (56s) 9s ago 5m 16.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (32s) 8s ago 5m 78.0M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (6m) 9s ago 6m 18.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (6m) 8s ago 6m 16.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283 running (96s) 8s ago 9m 131M - 19.3.0-4907-ga2782315 8801d2529e43 2372ccbce64c 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (22s) 9s ago 11m 490M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (11m) 9s ago 11m 56.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 c1073676c66c 2024-09-14T03:40:15.357 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (10m) 8s ago 10m 49.5M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 b6415a0de807 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (10m) 9s ago 10m 46.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 8e2894672cd8 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (51s) 9s ago 4m 8116k - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (27s) 8s ago 4m 7243k - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (9m) 9s ago 9m 132M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (9m) 9s ago 9m 137M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (8m) 9s ago 8m 104M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (8m) 9s ago 8m 102M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (7m) 8s ago 7m 182M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (7m) 8s ago 7m 129M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (7m) 8s ago 7m 87.4M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (6m) 8s ago 6m 90.0M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:40:15.358 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 starting - - - - 2024-09-14T03:40:15.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:15 smithi110 ceph-mon[35374]: pgmap v21: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 8.3 MiB/s wr, 62 op/s 2024-09-14T03:40:15.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:15 smithi110 ceph-mon[35374]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:40:15.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:15 smithi110 ceph-mon[35374]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:40:15.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:15 smithi110 ceph-mon[27365]: pgmap v21: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 8.3 MiB/s wr, 62 op/s 2024-09-14T03:40:15.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:15 smithi110 ceph-mon[27365]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:40:15.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:15 smithi110 ceph-mon[27365]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:40:15.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.196Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.197Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.197Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi195 (none))" 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.197Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.197Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.199Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.199Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.201Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.201Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.204Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.204Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.513µs 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.204Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:40:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.204Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.216Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.223Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.223Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.224Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=48.764µs wal_replay_duration=19.51752ms wbl_replay_duration=124ns total_replay_duration=19.597196ms 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.227Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.227Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.227Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.242Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.417363ms db_storage=856ns remote_storage=1.102µs web_handler=495ns query_engine=868ns scrape=448.568µs scrape_sd=107.305µs notify=9.32µs notify_sd=8.539µs rules=13.424132ms tracing=8.966µs 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.242Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:15.242Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:40:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 bash[74299]: e0fe3bbc1c08b6e4d30a0d40d2270143f4113effa04a84fb9604d01aab12b1a7 2024-09-14T03:40:15.462 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:15 smithi195 systemd[1]: Started Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:40:15.691 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:40:15.691 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:40:15.691 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:40:15.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:40:15.691 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:40:15.691 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:40:15.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:40:15.691 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 13, 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:40:15.692 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:40:15.969 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr" 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "2/20 daemons upgraded", 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading mgr daemons", 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:40:15.970 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:40:16.047 INFO:tasks.rados.rados.0.smithi110.stdout:894: done (4 left) 2024-09-14T03:40:16.047 INFO:tasks.rados.rados.0.smithi110.stdout:895: done (3 left) 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:896: done (2 left) 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:897: done (1 left) 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:898: done (0 left) 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:899: rollback oid 40 current snap is 87 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 40 to 80 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:900: rollback oid 16 current snap is 87 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 85 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:901: rollback oid 26 current snap is 87 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 69 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:902: read oid 18 snap -1 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:902: expect (ObjNum 278 snap 84 seq_num 278) 2024-09-14T03:40:16.048 INFO:tasks.rados.rados.0.smithi110.stdout:903: snap_create 2024-09-14T03:40:16.050 INFO:tasks.rados.rados.0.smithi110.stdout:899: finishing rollback tid 0 to smithi11075562-40 2024-09-14T03:40:16.052 INFO:tasks.rados.rados.0.smithi110.stdout:899: finishing rollback tid 1 to smithi11075562-40 2024-09-14T03:40:16.052 INFO:tasks.rados.rados.0.smithi110.stdout:899: finishing rollback tid 2 to smithi11075562-40 2024-09-14T03:40:16.052 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 391 (ObjNum 258 snap 79 seq_num 258) dirty exists 2024-09-14T03:40:16.053 INFO:tasks.rados.rados.0.smithi110.stdout:900: finishing rollback tid 0 to smithi11075562-16 2024-09-14T03:40:16.053 INFO:tasks.rados.rados.0.smithi110.stdout:900: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:40:16.053 INFO:tasks.rados.rados.0.smithi110.stdout:900: finishing rollback tid 2 to smithi11075562-16 2024-09-14T03:40:16.053 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 395 (ObjNum 234 snap 69 seq_num 234) dirty exists 2024-09-14T03:40:16.054 INFO:tasks.rados.rados.0.smithi110.stdout:901: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:40:16.055 INFO:tasks.rados.rados.0.smithi110.stdout:901: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:40:16.055 INFO:tasks.rados.rados.0.smithi110.stdout:901: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:40:16.055 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 138 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:40:16.078 INFO:tasks.rados.rados.0.smithi110.stdout:902: expect (ObjNum 278 snap 84 seq_num 278) 2024-09-14T03:40:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[35374]: from='client.25076 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[35374]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:40:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/3264953501' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[35374]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[27365]: from='client.25076 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[27365]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[27365]: from='client.? 172.21.15.110:0/3264953501' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:16 smithi110 ceph-mon[27365]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:40:16.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-mon[30695]: from='client.25076 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-mon[30695]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:40:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/3264953501' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:16.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-mon[30695]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:40:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:899: done (4 left) 2024-09-14T03:40:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:900: done (3 left) 2024-09-14T03:40:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:901: done (2 left) 2024-09-14T03:40:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:902: done (1 left) 2024-09-14T03:40:17.049 INFO:tasks.rados.rados.0.smithi110.stdout:903: done (0 left) 2024-09-14T03:40:17.050 INFO:tasks.rados.rados.0.smithi110.stdout:904: snap_create 2024-09-14T03:40:17.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: pgmap v24: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 7.9 MiB/s wr, 59 op/s 2024-09-14T03:40:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:40:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:17.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:17 smithi195 ceph-mon[30695]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:40:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: pgmap v24: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 7.9 MiB/s wr, 59 op/s 2024-09-14T03:40:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:40:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[35374]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: pgmap v24: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 7.9 MiB/s wr, 59 op/s 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:17.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:17.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:17.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:17.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:17 smithi110 ceph-mon[27365]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:40:18.053 INFO:tasks.rados.rados.0.smithi110.stdout:904: done (0 left) 2024-09-14T03:40:18.053 INFO:tasks.rados.rados.0.smithi110.stdout:905: delete oid 17 current snap is 89 2024-09-14T03:40:18.054 INFO:tasks.rados.rados.0.smithi110.stdout:905: done (0 left) 2024-09-14T03:40:18.054 INFO:tasks.rados.rados.0.smithi110.stdout:906: snap_create 2024-09-14T03:40:18.181 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:18 smithi195 ceph-mon[30695]: Upgrade: Updating mgr.x 2024-09-14T03:40:18.182 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:18 smithi195 ceph-mon[30695]: Deploying daemon mgr.x on smithi195 2024-09-14T03:40:18.182 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:18 smithi195 ceph-mon[30695]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:40:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:18 smithi110 ceph-mon[35374]: Upgrade: Updating mgr.x 2024-09-14T03:40:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:18 smithi110 ceph-mon[35374]: Deploying daemon mgr.x on smithi195 2024-09-14T03:40:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:18 smithi110 ceph-mon[35374]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:40:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:18 smithi110 ceph-mon[27365]: Upgrade: Updating mgr.x 2024-09-14T03:40:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:18 smithi110 ceph-mon[27365]: Deploying daemon mgr.x on smithi195 2024-09-14T03:40:18.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:18 smithi110 ceph-mon[27365]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:40:18.759 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:18 smithi195 systemd[1]: Stopping Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:40:19.037 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:18 smithi195 podman[74583]: 2024-09-14 03:40:18.757600762 +0000 UTC m=+0.133334468 container died 2372ccbce64cc806b61b00f1a2cc939df316b2d640c7c825e84d717b88440b8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, RELEASE=main-a278231, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T03:40:19.052 INFO:tasks.rados.rados.0.smithi110.stdout:906: done (0 left) 2024-09-14T03:40:19.053 INFO:tasks.rados.rados.0.smithi110.stdout:907: setattr oid 42 current snap is 90 2024-09-14T03:40:19.053 INFO:tasks.rados.rados.0.smithi110.stdout:908: delete oid 12 current snap is 90 2024-09-14T03:40:19.059 INFO:tasks.rados.rados.0.smithi110.stdout:908: done (1 left) 2024-09-14T03:40:19.059 INFO:tasks.rados.rados.0.smithi110.stdout:909: read oid 16 snap 79 2024-09-14T03:40:19.059 INFO:tasks.rados.rados.0.smithi110.stdout:909: expect (ObjNum 234 snap 69 seq_num 234) 2024-09-14T03:40:19.059 INFO:tasks.rados.rados.0.smithi110.stdout:910: rollback oid 44 current snap is 90 2024-09-14T03:40:19.059 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 44 to 81 2024-09-14T03:40:19.059 INFO:tasks.rados.rados.0.smithi110.stdout:911: rollback oid 1 current snap is 90 2024-09-14T03:40:19.060 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 1 to 81 2024-09-14T03:40:19.060 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 501 (ObjNum 279 snap 84 seq_num 279) dirty exists 2024-09-14T03:40:19.060 INFO:tasks.rados.rados.0.smithi110.stdout:907: done (3 left) 2024-09-14T03:40:19.060 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 90 2024-09-14T03:40:19.060 INFO:tasks.rados.rados.0.smithi110.stdout:912: seq_num 292 ranges {0=98304} 2024-09-14T03:40:19.060 INFO:tasks.rados.rados.0.smithi110.stdout:912: writing smithi11075562-20 from 0 to 98304 tid 1 2024-09-14T03:40:19.060 INFO:tasks.rados.rados.0.smithi110.stdout:913: rollback oid 12 current snap is 90 2024-09-14T03:40:19.060 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 84 2024-09-14T03:40:19.061 INFO:tasks.rados.rados.0.smithi110.stdout:914: read oid 23 snap -1 2024-09-14T03:40:19.061 INFO:tasks.rados.rados.0.smithi110.stdout:914: expect (ObjNum 280 snap 84 seq_num 280) 2024-09-14T03:40:19.061 INFO:tasks.rados.rados.0.smithi110.stdout:915: snap_create 2024-09-14T03:40:19.061 INFO:tasks.rados.rados.0.smithi110.stdout:913: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:40:19.061 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 542 (ObjNum 33 snap 0 seq_num 0) dirty dne 2024-09-14T03:40:19.063 INFO:tasks.rados.rados.0.smithi110.stdout:911: finishing rollback tid 0 to smithi11075562-1 2024-09-14T03:40:19.064 INFO:tasks.rados.rados.0.smithi110.stdout:910: finishing rollback tid 1 to smithi11075562-44 2024-09-14T03:40:19.064 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 376 (ObjNum 33 snap 0 seq_num 43181432) dirty dne 2024-09-14T03:40:19.065 INFO:tasks.rados.rados.0.smithi110.stdout:912: finishing write tid 1 to smithi11075562-20 2024-09-14T03:40:19.066 INFO:tasks.rados.rados.0.smithi110.stdout:912: finishing write tid 2 to smithi11075562-20 2024-09-14T03:40:19.066 INFO:tasks.rados.rados.0.smithi110.stdout:911: finishing rollback tid 1 to smithi11075562-1 2024-09-14T03:40:19.066 INFO:tasks.rados.rados.0.smithi110.stdout:911: finishing rollback tid 2 to smithi11075562-1 2024-09-14T03:40:19.067 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 505 (ObjNum 115 snap 28 seq_num 115) dirty exists 2024-09-14T03:40:19.067 INFO:tasks.rados.rados.0.smithi110.stdout:912: finishing write tid 3 to smithi11075562-20 2024-09-14T03:40:19.067 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 474 (ObjNum 292 snap 90 seq_num 292) dirty exists 2024-09-14T03:40:19.067 INFO:tasks.rados.rados.0.smithi110.stdout:912: left oid 20 (ObjNum 292 snap 90 seq_num 292) 2024-09-14T03:40:19.067 INFO:tasks.rados.rados.0.smithi110.stdout:909: expect (ObjNum 234 snap 69 seq_num 234) 2024-09-14T03:40:19.109 INFO:tasks.rados.rados.0.smithi110.stdout:914: expect (ObjNum 280 snap 84 seq_num 280) 2024-09-14T03:40:19.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:19 smithi195 ceph-mon[30695]: pgmap v27: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:19.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:19 smithi195 ceph-mon[30695]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:40:19.396 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 podman[74583]: 2024-09-14 03:40:19.152153415 +0000 UTC m=+0.527887114 container cleanup 2372ccbce64cc806b61b00f1a2cc939df316b2d640c7c825e84d717b88440b8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-09-14T03:40:19.396 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 bash[74583]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:40:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:19 smithi110 ceph-mon[35374]: pgmap v27: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:19 smithi110 ceph-mon[35374]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:40:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:19 smithi110 ceph-mon[27365]: pgmap v27: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:40:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:19 smithi110 ceph-mon[27365]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:40:19.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 podman[74596]: 2024-09-14 03:40:19.394764706 +0000 UTC m=+0.636177277 container remove 2372ccbce64cc806b61b00f1a2cc939df316b2d640c7c825e84d717b88440b8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-14T03:40:19.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:40:19.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-14T03:40:19.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 systemd[1]: Stopped Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:40:19.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x.service: Consumed 17.467s CPU time. 2024-09-14T03:40:20.053 INFO:tasks.rados.rados.0.smithi110.stdout:909: done (6 left) 2024-09-14T03:40:20.054 INFO:tasks.rados.rados.0.smithi110.stdout:910: done (5 left) 2024-09-14T03:40:20.054 INFO:tasks.rados.rados.0.smithi110.stdout:911: done (4 left) 2024-09-14T03:40:20.054 INFO:tasks.rados.rados.0.smithi110.stdout:912: done (3 left) 2024-09-14T03:40:20.054 INFO:tasks.rados.rados.0.smithi110.stdout:913: done (2 left) 2024-09-14T03:40:20.054 INFO:tasks.rados.rados.0.smithi110.stdout:914: done (1 left) 2024-09-14T03:40:20.054 INFO:tasks.rados.rados.0.smithi110.stdout:915: done (0 left) 2024-09-14T03:40:20.054 INFO:tasks.rados.rados.0.smithi110.stdout:916: snap_remove snap 89 2024-09-14T03:40:20.064 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 systemd[1]: Starting Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:40:20.064 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 podman[74726]: 2024-09-14 03:40:19.845802054 +0000 UTC m=+0.015866761 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:40:20.064 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:19 smithi195 podman[74726]: 2024-09-14 03:40:19.954413974 +0000 UTC m=+0.124478675 container create fbeb4897d8684a5b6de7209b7db4f23e04da1c84f14ec694d040cee0b099531a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:40:20.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:20 smithi195 podman[74726]: 2024-09-14 03:40:20.207031532 +0000 UTC m=+0.377096224 container init fbeb4897d8684a5b6de7209b7db4f23e04da1c84f14ec694d040cee0b099531a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T03:40:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:20 smithi195 podman[74726]: 2024-09-14 03:40:20.209757123 +0000 UTC m=+0.379821815 container start fbeb4897d8684a5b6de7209b7db4f23e04da1c84f14ec694d040cee0b099531a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-14T03:40:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:20 smithi195 ceph-mgr[74745]: -- 172.21.15.195:0/3328246410 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556bf43310e0 con 0x556bf430f000 2024-09-14T03:40:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:20 smithi195 bash[74726]: fbeb4897d8684a5b6de7209b7db4f23e04da1c84f14ec694d040cee0b099531a 2024-09-14T03:40:20.459 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:20 smithi195 systemd[1]: Started Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:40:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:916: done (0 left) 2024-09-14T03:40:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:917: rmattr oid 41 current snap is 91 2024-09-14T03:40:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:917: done (0 left) 2024-09-14T03:40:21.056 INFO:tasks.rados.rados.0.smithi110.stdout:918: snap_remove snap 56 2024-09-14T03:40:21.337 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:21 smithi195 ceph-mon[30695]: pgmap v30: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 12 KiB/s wr, 10 op/s 2024-09-14T03:40:21.337 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:21 smithi195 ceph-mon[30695]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:40:21.337 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:21 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:21.338 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:21 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:21.338 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:21 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[27365]: pgmap v30: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 12 KiB/s wr, 10 op/s 2024-09-14T03:40:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[27365]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:40:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[35374]: pgmap v30: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 12 KiB/s wr, 10 op/s 2024-09-14T03:40:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[35374]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:40:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:40:21] "GET /metrics HTTP/1.1" 200 36152 "" "Prometheus/2.51.0" 2024-09-14T03:40:22.067 INFO:tasks.rados.rados.0.smithi110.stdout:918: done (0 left) 2024-09-14T03:40:22.067 INFO:tasks.rados.rados.0.smithi110.stdout:919: setattr oid 11 current snap is 91 2024-09-14T03:40:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:920: rmattr oid 31 current snap is 91 2024-09-14T03:40:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:920: done (1 left) 2024-09-14T03:40:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:921: rollback oid 42 current snap is 91 2024-09-14T03:40:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 80 2024-09-14T03:40:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:922: rollback oid 38 current snap is 91 2024-09-14T03:40:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 72 2024-09-14T03:40:22.069 INFO:tasks.rados.rados.0.smithi110.stdout:923: snap_create 2024-09-14T03:40:22.075 INFO:tasks.rados.rados.0.smithi110.stdout:922: finishing rollback tid 0 to smithi11075562-38 2024-09-14T03:40:22.077 INFO:tasks.rados.rados.0.smithi110.stdout:922: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:40:22.080 INFO:tasks.rados.rados.0.smithi110.stdout:921: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:40:22.080 INFO:tasks.rados.rados.0.smithi110.stdout:922: finishing rollback tid 2 to smithi11075562-38 2024-09-14T03:40:22.080 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 241 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:40:22.083 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 387 (ObjNum 238 snap 69 seq_num 238) dirty exists 2024-09-14T03:40:22.084 INFO:tasks.rados.rados.0.smithi110.stdout:921: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:40:22.084 INFO:tasks.rados.rados.0.smithi110.stdout:921: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:40:22.084 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 521 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:40:22.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:22 smithi195 ceph-mon[30695]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:40:22.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:22 smithi110 ceph-mon[27365]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:40:22.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:22 smithi110 ceph-mon[35374]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:40:23.071 INFO:tasks.rados.rados.0.smithi110.stdout:919: done (3 left) 2024-09-14T03:40:23.071 INFO:tasks.rados.rados.0.smithi110.stdout:921: done (2 left) 2024-09-14T03:40:23.071 INFO:tasks.rados.rados.0.smithi110.stdout:922: done (1 left) 2024-09-14T03:40:23.071 INFO:tasks.rados.rados.0.smithi110.stdout:923: done (0 left) 2024-09-14T03:40:23.071 INFO:tasks.rados.rados.0.smithi110.stdout:924: rollback oid 43 current snap is 92 2024-09-14T03:40:23.071 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 43 to 67 2024-09-14T03:40:23.072 INFO:tasks.rados.rados.0.smithi110.stdout:925: snap_create 2024-09-14T03:40:23.076 INFO:tasks.rados.rados.0.smithi110.stdout:924: finishing rollback tid 0 to smithi11075562-43 2024-09-14T03:40:23.076 INFO:tasks.rados.rados.0.smithi110.stdout:924: finishing rollback tid 1 to smithi11075562-43 2024-09-14T03:40:23.076 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 483 (ObjNum 33 snap 0 seq_num 65430720) dirty dne 2024-09-14T03:40:23.438 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:23 smithi195 ceph-mon[30695]: pgmap v33: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 12 KiB/s wr, 10 op/s 2024-09-14T03:40:23.438 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:23 smithi195 ceph-mon[30695]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:40:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:23 smithi110 ceph-mon[35374]: pgmap v33: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 12 KiB/s wr, 10 op/s 2024-09-14T03:40:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:23 smithi110 ceph-mon[35374]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:40:23.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:23 smithi110 ceph-mon[27365]: pgmap v33: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 12 KiB/s wr, 10 op/s 2024-09-14T03:40:23.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:23 smithi110 ceph-mon[27365]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:40:24.078 INFO:tasks.rados.rados.0.smithi110.stdout:924: done (1 left) 2024-09-14T03:40:24.078 INFO:tasks.rados.rados.0.smithi110.stdout:925: done (0 left) 2024-09-14T03:40:24.078 INFO:tasks.rados.rados.0.smithi110.stdout:926: rmattr oid 29 current snap is 93 2024-09-14T03:40:24.078 INFO:tasks.rados.rados.0.smithi110.stdout:926: done (0 left) 2024-09-14T03:40:24.078 INFO:tasks.rados.rados.0.smithi110.stdout:927: snap_create 2024-09-14T03:40:24.388 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:24 smithi195 ceph-mon[30695]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:40:24.389 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:24 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:24.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:24 smithi110 ceph-mon[35374]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:40:24.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:24 smithi110 ceph-mon[27365]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:40:25.049 INFO:tasks.rados.rados.0.smithi110.stdout:927: done (0 left) 2024-09-14T03:40:25.049 INFO:tasks.rados.rados.0.smithi110.stdout:928: delete oid 33 current snap is 94 2024-09-14T03:40:25.054 INFO:tasks.rados.rados.0.smithi110.stdout:928: done (0 left) 2024-09-14T03:40:25.054 INFO:tasks.rados.rados.0.smithi110.stdout:929: setattr oid 48 current snap is 94 2024-09-14T03:40:25.055 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 94 2024-09-14T03:40:25.055 INFO:tasks.rados.rados.0.smithi110.stdout:930: seq_num 294 ranges {2252800=131072} 2024-09-14T03:40:25.068 INFO:tasks.rados.rados.0.smithi110.stdout:930: writing smithi11075562-2 from 2252800 to 2383872 tid 1 2024-09-14T03:40:25.068 INFO:tasks.rados.rados.0.smithi110.stdout:931: read oid 37 snap -1 2024-09-14T03:40:25.068 INFO:tasks.rados.rados.0.smithi110.stdout:931: expect (ObjNum 285 snap 85 seq_num 285) 2024-09-14T03:40:25.068 INFO:tasks.rados.rados.0.smithi110.stdout:932: copy_from oid 40 from oid 33 current snap is 94 2024-09-14T03:40:25.069 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 388 (ObjNum 214 snap 65 seq_num 214) dirty exists 2024-09-14T03:40:25.069 INFO:tasks.rados.rados.0.smithi110.stdout:929: done (3 left) 2024-09-14T03:40:25.069 INFO:tasks.rados.rados.0.smithi110.stdout:933: snap_remove snap 64 2024-09-14T03:40:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[35374]: pgmap v36: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 24 KiB/s wr, 15 op/s 2024-09-14T03:40:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[35374]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:40:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[35374]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:40:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[27365]: pgmap v36: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 24 KiB/s wr, 15 op/s 2024-09-14T03:40:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[27365]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:40:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:25 smithi110 ceph-mon[27365]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:40:25.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-mon[30695]: pgmap v36: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 24 KiB/s wr, 15 op/s 2024-09-14T03:40:25.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-mon[30695]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:40:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-mon[30695]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:40:25.814 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:25.814 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:25.815 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: [14/Sep/2024:03:40:25] ENGINE Bus STARTING 2024-09-14T03:40:25.815 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: CherryPy Checker: 2024-09-14T03:40:25.815 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: The Application mounted at '' has an empty config. 2024-09-14T03:40:25.815 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: 2024-09-14T03:40:25.815 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: [14/Sep/2024:03:40:25] ENGINE Serving on http://:::9283 2024-09-14T03:40:25.815 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: [14/Sep/2024:03:40:25] ENGINE Bus STARTED 2024-09-14T03:40:26.052 INFO:tasks.rados.rados.0.smithi110.stdout:933: done (3 left) 2024-09-14T03:40:26.052 INFO:tasks.rados.rados.0.smithi110.stdout:934: setattr oid 27 current snap is 94 2024-09-14T03:40:26.052 INFO:tasks.rados.rados.0.smithi110.stdout:932: finishing copy_from to smithi11075562-40 2024-09-14T03:40:26.052 INFO:tasks.rados.rados.0.smithi110.stdout:932: got expected ENOENT (src dne) 2024-09-14T03:40:26.052 INFO:tasks.rados.rados.0.smithi110.stdout:932: finishing copy_from racing read to smithi11075562-40 2024-09-14T03:40:26.052 INFO:tasks.rados.rados.0.smithi110.stdout:930: finishing write tid 1 to smithi11075562-2 2024-09-14T03:40:26.052 INFO:tasks.rados.rados.0.smithi110.stdout:930: finishing write tid 2 to smithi11075562-2 2024-09-14T03:40:26.053 INFO:tasks.rados.rados.0.smithi110.stdout:930: finishing write tid 3 to smithi11075562-2 2024-09-14T03:40:26.053 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 327 (ObjNum 294 snap 94 seq_num 294) dirty exists 2024-09-14T03:40:26.053 INFO:tasks.rados.rados.0.smithi110.stdout:930: left oid 2 (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:40:26.053 INFO:tasks.rados.rados.0.smithi110.stdout:931: expect (ObjNum 285 snap 85 seq_num 285) 2024-09-14T03:40:26.122 INFO:tasks.rados.rados.0.smithi110.stdout:930: done (3 left) 2024-09-14T03:40:26.122 INFO:tasks.rados.rados.0.smithi110.stdout:931: done (2 left) 2024-09-14T03:40:26.122 INFO:tasks.rados.rados.0.smithi110.stdout:932: done (1 left) 2024-09-14T03:40:26.122 INFO:tasks.rados.rados.0.smithi110.stdout:935: snap_create 2024-09-14T03:40:26.125 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 98 (ObjNum 65 snap 0 seq_num 1666143576) dirty exists 2024-09-14T03:40:26.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: pgmap v39: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 24 KiB/s wr, 15 op/s 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: Standby manager daemon x restarted 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: Standby manager daemon x started 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[27365]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: pgmap v39: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 24 KiB/s wr, 15 op/s 2024-09-14T03:40:26.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:26.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:26.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: Standby manager daemon x restarted 2024-09-14T03:40:26.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: Standby manager daemon x started 2024-09-14T03:40:26.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:26.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:26.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:26.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:26.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:26 smithi110 ceph-mon[35374]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:40:26.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: pgmap v39: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 24 KiB/s wr, 15 op/s 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: Standby manager daemon x restarted 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: Standby manager daemon x started 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/3138122189' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:26.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-mon[30695]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:40:27.080 INFO:tasks.rados.rados.0.smithi110.stdout:934: done (1 left) 2024-09-14T03:40:27.080 INFO:tasks.rados.rados.0.smithi110.stdout:935: done (0 left) 2024-09-14T03:40:27.080 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 95 2024-09-14T03:40:27.080 INFO:tasks.rados.rados.0.smithi110.stdout:936: seq_num 295 ranges {2383872=147456} 2024-09-14T03:40:27.093 INFO:tasks.rados.rados.0.smithi110.stdout:936: writing smithi11075562-37 from 2383872 to 2531328 tid 1 2024-09-14T03:40:27.093 INFO:tasks.rados.rados.0.smithi110.stdout:937: snap_create 2024-09-14T03:40:27.098 INFO:tasks.rados.rados.0.smithi110.stdout:936: finishing write tid 1 to smithi11075562-37 2024-09-14T03:40:27.098 INFO:tasks.rados.rados.0.smithi110.stdout:936: finishing write tid 2 to smithi11075562-37 2024-09-14T03:40:27.098 INFO:tasks.rados.rados.0.smithi110.stdout:936: finishing write tid 3 to smithi11075562-37 2024-09-14T03:40:27.098 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 570 (ObjNum 295 snap 95 seq_num 295) dirty exists 2024-09-14T03:40:27.098 INFO:tasks.rados.rados.0.smithi110.stdout:936: left oid 37 (ObjNum 295 snap 95 seq_num 295) 2024-09-14T03:40:27.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:26 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:27.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:27 smithi110 ceph-mon[35374]: mgrmap e37: y(active, since 25s), standbys: x 2024-09-14T03:40:27.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:27 smithi110 ceph-mon[35374]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:40:27.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:27 smithi110 ceph-mon[27365]: mgrmap e37: y(active, since 25s), standbys: x 2024-09-14T03:40:27.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:27 smithi110 ceph-mon[27365]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:40:27.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:27 smithi195 ceph-mon[30695]: mgrmap e37: y(active, since 25s), standbys: x 2024-09-14T03:40:27.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:27 smithi195 ceph-mon[30695]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:40:28.084 INFO:tasks.rados.rados.0.smithi110.stdout:936: done (1 left) 2024-09-14T03:40:28.085 INFO:tasks.rados.rados.0.smithi110.stdout:937: done (0 left) 2024-09-14T03:40:28.085 INFO:tasks.rados.rados.0.smithi110.stdout:938: read oid 11 snap 93 2024-09-14T03:40:28.085 INFO:tasks.rados.rados.0.smithi110.stdout:938: expect (ObjNum 238 snap 69 seq_num 238) 2024-09-14T03:40:28.085 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 96 2024-09-14T03:40:28.085 INFO:tasks.rados.rados.0.smithi110.stdout:939: seq_num 296 ranges {6012928=163840} 2024-09-14T03:40:28.119 INFO:tasks.rados.rados.0.smithi110.stdout:939: writing smithi11075562-39 from 6012928 to 6176768 tid 1 2024-09-14T03:40:28.119 INFO:tasks.rados.rados.0.smithi110.stdout:940: snap_remove snap 94 2024-09-14T03:40:29.089 INFO:tasks.rados.rados.0.smithi110.stdout:940: done (2 left) 2024-09-14T03:40:29.089 INFO:tasks.rados.rados.0.smithi110.stdout:941: setattr oid 41 current snap is 96 2024-09-14T03:40:29.090 INFO:tasks.rados.rados.0.smithi110.stdout:938: expect (ObjNum 238 snap 69 seq_num 238) 2024-09-14T03:40:29.197 INFO:tasks.rados.rados.0.smithi110.stdout:939: finishing write tid 1 to smithi11075562-39 2024-09-14T03:40:29.197 INFO:tasks.rados.rados.0.smithi110.stdout:939: finishing write tid 2 to smithi11075562-39 2024-09-14T03:40:29.198 INFO:tasks.rados.rados.0.smithi110.stdout:939: finishing write tid 3 to smithi11075562-39 2024-09-14T03:40:29.198 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 492 (ObjNum 296 snap 96 seq_num 296) dirty exists 2024-09-14T03:40:29.198 INFO:tasks.rados.rados.0.smithi110.stdout:939: left oid 39 (ObjNum 296 snap 96 seq_num 296) 2024-09-14T03:40:29.198 INFO:tasks.rados.rados.0.smithi110.stdout:938: done (2 left) 2024-09-14T03:40:29.198 INFO:tasks.rados.rados.0.smithi110.stdout:939: done (1 left) 2024-09-14T03:40:29.198 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 96 2024-09-14T03:40:29.198 INFO:tasks.rados.rados.0.smithi110.stdout:942: seq_num 297 ranges {4521984=180224} 2024-09-14T03:40:29.223 INFO:tasks.rados.rados.0.smithi110.stdout:942: writing smithi11075562-32 from 4521984 to 4702208 tid 1 2024-09-14T03:40:29.223 INFO:tasks.rados.rados.0.smithi110.stdout:943: delete oid 43 current snap is 96 2024-09-14T03:40:29.223 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 495 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:40:29.224 INFO:tasks.rados.rados.0.smithi110.stdout:941: done (2 left) 2024-09-14T03:40:29.224 INFO:tasks.rados.rados.0.smithi110.stdout:943: done (1 left) 2024-09-14T03:40:29.225 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 96 2024-09-14T03:40:29.225 INFO:tasks.rados.rados.0.smithi110.stdout:944: seq_num 299 ranges {1196032=212992} 2024-09-14T03:40:29.232 INFO:tasks.rados.rados.0.smithi110.stdout:944: writing smithi11075562-48 from 1196032 to 1409024 tid 1 2024-09-14T03:40:29.232 INFO:tasks.rados.rados.0.smithi110.stdout:945: snap_create 2024-09-14T03:40:29.232 INFO:tasks.rados.rados.0.smithi110.stdout:942: finishing write tid 1 to smithi11075562-32 2024-09-14T03:40:29.232 INFO:tasks.rados.rados.0.smithi110.stdout:942: finishing write tid 2 to smithi11075562-32 2024-09-14T03:40:29.232 INFO:tasks.rados.rados.0.smithi110.stdout:942: finishing write tid 3 to smithi11075562-32 2024-09-14T03:40:29.233 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 479 (ObjNum 297 snap 96 seq_num 297) dirty exists 2024-09-14T03:40:29.233 INFO:tasks.rados.rados.0.smithi110.stdout:942: left oid 32 (ObjNum 297 snap 96 seq_num 297) 2024-09-14T03:40:29.235 INFO:tasks.rados.rados.0.smithi110.stdout:944: finishing write tid 1 to smithi11075562-48 2024-09-14T03:40:29.235 INFO:tasks.rados.rados.0.smithi110.stdout:944: finishing write tid 2 to smithi11075562-48 2024-09-14T03:40:29.235 INFO:tasks.rados.rados.0.smithi110.stdout:944: finishing write tid 3 to smithi11075562-48 2024-09-14T03:40:29.236 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 390 (ObjNum 299 snap 96 seq_num 299) dirty exists 2024-09-14T03:40:29.236 INFO:tasks.rados.rados.0.smithi110.stdout:944: left oid 48 (ObjNum 299 snap 96 seq_num 299) 2024-09-14T03:40:29.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: pgmap v42: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-14T03:40:29.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:40:29.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:40:29.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: pgmap v42: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:40:29.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:40:29.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:40:29.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:40:29.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:40:29.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:29 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:40:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: pgmap v42: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-14T03:40:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:40:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:29.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:29.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:29.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:40:29.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:40:29.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:40:29.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:40:29.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:40:29.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:29 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:40:30.095 INFO:tasks.rados.rados.0.smithi110.stdout:942: done (2 left) 2024-09-14T03:40:30.095 INFO:tasks.rados.rados.0.smithi110.stdout:944: done (1 left) 2024-09-14T03:40:30.095 INFO:tasks.rados.rados.0.smithi110.stdout:945: done (0 left) 2024-09-14T03:40:30.095 INFO:tasks.rados.rados.0.smithi110.stdout:946: snap_remove snap 80 2024-09-14T03:40:30.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[27365]: Upgrade: Setting container_image for all mgr 2024-09-14T03:40:30.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[27365]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:40:30.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[27365]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:40:30.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:30.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:30.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:30.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:30.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[35374]: Upgrade: Setting container_image for all mgr 2024-09-14T03:40:30.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[35374]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:40:30.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[35374]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:40:30.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:30.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:30.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:30.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:30 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:30.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:30 smithi195 ceph-mon[30695]: Upgrade: Setting container_image for all mgr 2024-09-14T03:40:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:30 smithi195 ceph-mon[30695]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:40:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:30 smithi195 ceph-mon[30695]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:40:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:30 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' 2024-09-14T03:40:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:30 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:30.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:30 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:30.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:30 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:946: done (0 left) 2024-09-14T03:40:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:947: rmattr oid 46 current snap is 97 2024-09-14T03:40:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:947: done (0 left) 2024-09-14T03:40:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 97 2024-09-14T03:40:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:948: seq_num 300 ranges {0=229376} 2024-09-14T03:40:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:948: writing smithi11075562-21 from 0 to 229376 tid 1 2024-09-14T03:40:31.097 INFO:tasks.rados.rados.0.smithi110.stdout:949: delete oid 39 current snap is 97 2024-09-14T03:40:31.109 INFO:tasks.rados.rados.0.smithi110.stdout:948: finishing write tid 1 to smithi11075562-21 2024-09-14T03:40:31.109 INFO:tasks.rados.rados.0.smithi110.stdout:948: finishing write tid 2 to smithi11075562-21 2024-09-14T03:40:31.109 INFO:tasks.rados.rados.0.smithi110.stdout:949: done (1 left) 2024-09-14T03:40:31.109 INFO:tasks.rados.rados.0.smithi110.stdout:950: read oid 49 snap -1 2024-09-14T03:40:31.109 INFO:tasks.rados.rados.0.smithi110.stdout:950: expect deleted 2024-09-14T03:40:31.110 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 97 2024-09-14T03:40:31.110 INFO:tasks.rados.rados.0.smithi110.stdout:951: seq_num 302 ranges {3137536=262144} 2024-09-14T03:40:31.128 INFO:tasks.rados.rados.0.smithi110.stdout:951: writing smithi11075562-28 from 3137536 to 3399680 tid 1 2024-09-14T03:40:31.128 INFO:tasks.rados.rados.0.smithi110.stdout:952: read oid 39 snap -1 2024-09-14T03:40:31.128 INFO:tasks.rados.rados.0.smithi110.stdout:952: expect deleted 2024-09-14T03:40:31.128 INFO:tasks.rados.rados.0.smithi110.stdout:953: rmattr oid 12 current snap is 97 2024-09-14T03:40:31.128 INFO:tasks.rados.rados.0.smithi110.stdout:953: done (4 left) 2024-09-14T03:40:31.128 INFO:tasks.rados.rados.0.smithi110.stdout:954: delete oid 3 current snap is 97 2024-09-14T03:40:31.128 INFO:tasks.rados.rados.0.smithi110.stdout:948: finishing write tid 3 to smithi11075562-21 2024-09-14T03:40:31.128 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 498 (ObjNum 300 snap 97 seq_num 300) dirty exists 2024-09-14T03:40:31.129 INFO:tasks.rados.rados.0.smithi110.stdout:948: left oid 21 (ObjNum 300 snap 97 seq_num 300) 2024-09-14T03:40:31.130 INFO:tasks.rados.rados.0.smithi110.stdout:951: finishing write tid 1 to smithi11075562-28 2024-09-14T03:40:31.130 INFO:tasks.rados.rados.0.smithi110.stdout:951: finishing write tid 2 to smithi11075562-28 2024-09-14T03:40:31.131 INFO:tasks.rados.rados.0.smithi110.stdout:948: done (4 left) 2024-09-14T03:40:31.131 INFO:tasks.rados.rados.0.smithi110.stdout:950: done (3 left) 2024-09-14T03:40:31.131 INFO:tasks.rados.rados.0.smithi110.stdout:952: done (2 left) 2024-09-14T03:40:31.131 INFO:tasks.rados.rados.0.smithi110.stdout:954: done (1 left) 2024-09-14T03:40:31.131 INFO:tasks.rados.rados.0.smithi110.stdout:955: copy_from oid 32 from oid 19 current snap is 97 2024-09-14T03:40:31.131 INFO:tasks.rados.rados.0.smithi110.stdout:956: snap_remove snap 69 2024-09-14T03:40:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-mon[27365]: pgmap v45: 121 pgs: 121 active+clean; 261 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 68 KiB/s wr, 13 op/s 2024-09-14T03:40:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-mon[27365]: Upgrade: Updating mon.a 2024-09-14T03:40:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-mon[27365]: Deploying daemon mon.a on smithi110 2024-09-14T03:40:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-mon[27365]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:40:31.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-mon[35374]: pgmap v45: 121 pgs: 121 active+clean; 261 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 68 KiB/s wr, 13 op/s 2024-09-14T03:40:31.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-mon[35374]: Upgrade: Updating mon.a 2024-09-14T03:40:31.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-mon[35374]: Deploying daemon mon.a on smithi110 2024-09-14T03:40:31.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-mon[35374]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:40:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:31 smithi195 ceph-mon[30695]: pgmap v45: 121 pgs: 121 active+clean; 261 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 68 KiB/s wr, 13 op/s 2024-09-14T03:40:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:31 smithi195 ceph-mon[30695]: Upgrade: Updating mon.a 2024-09-14T03:40:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:31 smithi195 ceph-mon[30695]: Deploying daemon mon.a on smithi110 2024-09-14T03:40:31.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:31 smithi195 ceph-mon[30695]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:40:31.857 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:40:31] "GET /metrics HTTP/1.1" 200 36152 "" "Prometheus/2.51.0" 2024-09-14T03:40:32.110 INFO:tasks.rados.rados.0.smithi110.stdout:951: finishing write tid 3 to smithi11075562-28 2024-09-14T03:40:32.110 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 340 (ObjNum 302 snap 97 seq_num 302) dirty exists 2024-09-14T03:40:32.110 INFO:tasks.rados.rados.0.smithi110.stdout:951: left oid 28 (ObjNum 302 snap 97 seq_num 302) 2024-09-14T03:40:32.111 INFO:tasks.rados.rados.0.smithi110.stdout:955: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:40:32.111 INFO:tasks.rados.rados.0.smithi110.stdout:955: finishing copy_from to smithi11075562-32 2024-09-14T03:40:32.111 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 369 (ObjNum 235 snap 69 seq_num 235) dirty exists 2024-09-14T03:40:32.111 INFO:tasks.rados.rados.0.smithi110.stdout:951: done (2 left) 2024-09-14T03:40:32.111 INFO:tasks.rados.rados.0.smithi110.stdout:955: done (1 left) 2024-09-14T03:40:32.111 INFO:tasks.rados.rados.0.smithi110.stdout:956: done (0 left) 2024-09-14T03:40:32.112 INFO:tasks.rados.rados.0.smithi110.stdout:957: snap_create 2024-09-14T03:40:32.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:31 smithi110 systemd[1]: Stopping Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:40:32.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-mon[27365]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:40:32.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-mon[27365]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:32.122 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-mon[35374]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:40:32.122 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:32.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:32] ENGINE Bus STOPPING 2024-09-14T03:40:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[27361]: 2024-09-14T03:40:32.121+0000 7f6566f4b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:40:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[27361]: 2024-09-14T03:40:32.121+0000 7f6566f4b640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-14T03:40:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 podman[86244]: 2024-09-14 03:40:32.152634741 +0000 UTC m=+0.146024405 container died c1073676c66c24575c3a0a0a56ec31782bf4290020321ad3b4f76914a50aa092 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-14T03:40:32.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:32 smithi195 ceph-mon[30695]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:40:32.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:32 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:32.791 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 podman[86244]: 2024-09-14 03:40:32.547711277 +0000 UTC m=+0.541100940 container cleanup c1073676c66c24575c3a0a0a56ec31782bf4290020321ad3b4f76914a50aa092 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-14T03:40:32.792 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 bash[86244]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:40:32.792 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:40:32.792 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:32] ENGINE Bus STOPPED 2024-09-14T03:40:32.792 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:32] ENGINE Bus STARTING 2024-09-14T03:40:32.792 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:32] ENGINE Serving on http://:::9283 2024-09-14T03:40:32.793 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:32] ENGINE Bus STARTED 2024-09-14T03:40:33.123 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 podman[86256]: 2024-09-14 03:40:32.790944608 +0000 UTC m=+0.636076840 container remove c1073676c66c24575c3a0a0a56ec31782bf4290020321ad3b4f76914a50aa092 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:40:33.123 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T03:40:33.123 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 systemd[1]: Stopped Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:40:33.123 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:32 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a.service: Consumed 13.093s CPU time. 2024-09-14T03:40:33.123 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 systemd[1]: Starting Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:40:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 podman[86404]: 2024-09-14 03:40:33.236906262 +0000 UTC m=+0.018048586 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:40:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 podman[86404]: 2024-09-14 03:40:33.358953618 +0000 UTC m=+0.140095924 container create c325cb2844f41afa7669ff7ca8517c47c716a8479eb3b5110e63a1b185d9e405 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-14T03:40:33.758 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 podman[86404]: 2024-09-14 03:40:33.603394278 +0000 UTC m=+0.384536581 container init c325cb2844f41afa7669ff7ca8517c47c716a8479eb3b5110e63a1b185d9e405 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T03:40:33.758 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 podman[86404]: 2024-09-14 03:40:33.606578329 +0000 UTC m=+0.387720635 container start c325cb2844f41afa7669ff7ca8517c47c716a8479eb3b5110e63a1b185d9e405 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-14T03:40:33.758 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:40:33.758 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 2 2024-09-14T03:40:33.758 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: pidfile_write: ignore empty --pid-file 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: load: jerasure load: lrc 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Git sha 0 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: DB SUMMARY 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: DB Session ID: 94OC3R5T8AJIC5U80MP8 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1179 Bytes 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 4544851 ; 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:40:33.759 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.env: 0x55903e734120 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.info_log: 0x55903fe9f700 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.statistics: (nil) 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.use_fsync: 0 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:40:33.760 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.db_log_dir: 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.wal_dir: 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:40:33.761 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.write_buffer_manager: 0x55903fea21e0 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:40:33.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:40:33.763 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.unordered_write: 0 2024-09-14T03:40:33.763 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:40:33.763 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:40:33.763 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:40:33.763 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:40:33.763 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.row_cache: None 2024-09-14T03:40:33.763 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.wal_filter: None 2024-09-14T03:40:33.763 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.wal_compression: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:40:33.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:40:33.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_open_files: -1 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Compression algorithms supported: 2024-09-14T03:40:33.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: kZSTD supported: 0 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:40:33.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.merge_operator: 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_filter: None 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55903fe9f320) 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:40:33.769 INFO:journalctl@ceph.mon.a.smithi110.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_type: 0 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_block_index_type: 0 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_shortening: 1 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: checksum: 4 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: no_block_cache: 0 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache: 0x55903fe93610 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_options: 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: capacity : 536870912 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: num_shard_bits : 4 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: strict_capacity_limit : 0 2024-09-14T03:40:33.770 INFO:journalctl@ceph.mon.a.smithi110.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_compressed: (nil) 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: persistent_cache: (nil) 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_size: 4096 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_size_deviation: 10 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_restart_interval: 16 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_block_restart_interval: 1 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: metadata_block_size: 4096 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: partition_filters: 0 2024-09-14T03:40:33.771 INFO:journalctl@ceph.mon.a.smithi110.stdout: use_delta_encoding: 1 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: filter_policy: bloomfilter 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: whole_key_filtering: 1 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: verify_compression: 0 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: format_version: 5 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: enable_index_compression: 1 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_align: 0 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: max_auto_readahead_size: 262144 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: prepopulate_block_cache: 0 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:40:33.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression: NoCompression 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.num_levels: 7 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:40:33.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:40:33.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:40:33.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:40:33.776 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:40:33.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:40:33.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.ttl: 2592000 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:40:33.779 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12844, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 2e53e134-9268-490a-b647-d5e26da85c9a 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285233633190, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-14T03:40:33.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285233645788, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 4274473, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12845, "largest_seqno": 14940, "table_properties": {"data_size": 4263301, "index_size": 7822, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2309, "raw_key_size": 22311, "raw_average_key_size": 24, "raw_value_size": 4242333, "raw_average_value_size": 4626, "num_data_blocks": 342, "num_entries": 917, "num_filter_entries": 917, "num_deletions": 38, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285233, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "2e53e134-9268-490a-b647-d5e26da85c9a", "db_session_id": "94OC3R5T8AJIC5U80MP8", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285233646003, "job": 1, "event": "recovery_finished"} 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55903fec2e00 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: DB pointer 0x55903ffc4000 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** DB Stats ** 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:40:33.781 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: L0 1/0 4.08 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 645.0 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-14T03:40:33.782 INFO:journalctl@ceph.mon.a.smithi110.stdout: L6 1/0 7.79 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: Sum 2/0 11.86 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 645.0 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 645.0 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 645.0 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:40:33.783 INFO:journalctl@ceph.mon.a.smithi110.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative compaction: 0.00 GB write, 264.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval compaction: 0.00 GB write, 264.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: Block cache BinnedLRUCache@0x55903fe93610#2 capacity: 512.00 MB usage: 10.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.53 KB,0.000482798%) IndexBlock(1,8.08 KB,0.00154078%) Misc(1,0.00 KB,0%) 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: starting mon.a rank 0 at public addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] at bind addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:40:33.784 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: mon.a@-1(???) e3 preinit fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: mon.a@-1(???).mds e7 new map 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: mon.a@-1(???).mds e7 print_map 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: e7 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: legacy client fscid: 1 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: Filesystem 'cephfs' (1) 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: fs_name cephfs 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: epoch 7 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:40:33.785 INFO:journalctl@ceph.mon.a.smithi110.stdout: created 2024-09-14T03:34:04.160374+0000 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: modified 2024-09-14T03:34:15.891707+0000 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: tableserver 0 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: root 0 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: session_timeout 60 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: session_autoclose 300 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: max_file_size 1099511627776 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: max_xattr_size 65536 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: required_client_features {} 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: last_failure 0 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: last_failure_osd_epoch 0 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: max_mds 1 2024-09-14T03:40:33.786 INFO:journalctl@ceph.mon.a.smithi110.stdout: in 0 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: up {0=24311} 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: failed 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: damaged 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: stopped 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_pools [3] 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: metadata_pool 2 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: inline_data disabled 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: balancer 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: bal_rank_mask -1 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: standby_count_wanted 0 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: [mds.a{0:24311} state up:active seq 6 addr [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:40:33.787 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout: Standby daemons: 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout: [mds.b{-1:24310} state up:standby seq 1 addr [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: mon.a@-1(???).osd e247 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: mon.a@-1(???).osd e247 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: mon.a@-1(???).osd e247 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: mon.a@-1(???).osd e247 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 ceph-mon[86423]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 bash[86404]: c325cb2844f41afa7669ff7ca8517c47c716a8479eb3b5110e63a1b185d9e405 2024-09-14T03:40:33.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:33 smithi110 systemd[1]: Started Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:40:34.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:34 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:36.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:36.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:36.740 INFO:tasks.rados.rados.0.smithi110.stdout:957: done (0 left) 2024-09-14T03:40:36.740 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 98 2024-09-14T03:40:36.740 INFO:tasks.rados.rados.0.smithi110.stdout:958: seq_num 304 ranges {229376=294912} 2024-09-14T03:40:36.742 INFO:tasks.rados.rados.0.smithi110.stdout:958: writing smithi11075562-21 from 229376 to 524288 tid 1 2024-09-14T03:40:36.742 INFO:tasks.rados.rados.0.smithi110.stdout:959: rollback oid 42 current snap is 98 2024-09-14T03:40:36.742 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 81 2024-09-14T03:40:36.742 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 98 2024-09-14T03:40:36.743 INFO:tasks.rados.rados.0.smithi110.stdout:960: seq_num 305 ranges {1245184=311296} 2024-09-14T03:40:36.750 INFO:tasks.rados.rados.0.smithi110.stdout:960: writing smithi11075562-47 from 1245184 to 1556480 tid 1 2024-09-14T03:40:36.750 INFO:tasks.rados.rados.0.smithi110.stdout:961: copy_from oid 13 from oid 7 current snap is 98 2024-09-14T03:40:36.750 INFO:tasks.rados.rados.0.smithi110.stdout:958: finishing write tid 1 to smithi11075562-21 2024-09-14T03:40:36.750 INFO:tasks.rados.rados.0.smithi110.stdout:958: finishing write tid 2 to smithi11075562-21 2024-09-14T03:40:36.750 INFO:tasks.rados.rados.0.smithi110.stdout:959: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:959: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:959: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 555 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:958: finishing write tid 3 to smithi11075562-21 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 517 (ObjNum 304 snap 98 seq_num 304) dirty exists 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:958: left oid 21 (ObjNum 304 snap 98 seq_num 304) 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:958: done (3 left) 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:959: done (2 left) 2024-09-14T03:40:36.751 INFO:tasks.rados.rados.0.smithi110.stdout:962: snap_create 2024-09-14T03:40:36.752 INFO:tasks.rados.rados.0.smithi110.stdout:961: finishing copy_from to smithi11075562-13 2024-09-14T03:40:36.752 INFO:tasks.rados.rados.0.smithi110.stdout:961: got expected ENOENT (src dne) 2024-09-14T03:40:36.752 INFO:tasks.rados.rados.0.smithi110.stdout:961: finishing copy_from racing read to smithi11075562-13 2024-09-14T03:40:36.752 INFO:tasks.rados.rados.0.smithi110.stdout:960: finishing write tid 1 to smithi11075562-47 2024-09-14T03:40:36.752 INFO:tasks.rados.rados.0.smithi110.stdout:960: finishing write tid 2 to smithi11075562-47 2024-09-14T03:40:36.761 INFO:tasks.rados.rados.0.smithi110.stdout:960: finishing write tid 3 to smithi11075562-47 2024-09-14T03:40:36.762 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 585 (ObjNum 305 snap 98 seq_num 305) dirty exists 2024-09-14T03:40:36.762 INFO:tasks.rados.rados.0.smithi110.stdout:960: left oid 47 (ObjNum 305 snap 98 seq_num 305) 2024-09-14T03:40:37.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ignoring --setuser ceph since I am not root 2024-09-14T03:40:37.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ignoring --setgroup ceph since I am not root 2024-09-14T03:40:37.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mgr[78678]: -- 172.21.15.110:0/2032809436 <== mon.0 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56201da4b0e0 con 0x56201da29400 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: pgmap v50: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 605 KiB/s wr, 10 op/s 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: pgmap v51: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 423 KiB/s wr, 7 op/s 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: mon.a calling monitor election 2024-09-14T03:40:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: monmap epoch 3 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: last_changed 2024-09-14T03:30:13.577071+0000 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: min_mon_release 17 (quincy) 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: election_strategy: 1 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-14T03:40:37.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: overall HEALTH_OK 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: from='mgr.25019 ' entity='' 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[35374]: mgrmap e38: y(active, since 35s), standbys: x 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: pgmap v50: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 605 KiB/s wr, 10 op/s 2024-09-14T03:40:37.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: pgmap v51: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 423 KiB/s wr, 7 op/s 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: mon.a calling monitor election 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: monmap epoch 3 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: last_changed 2024-09-14T03:30:13.577071+0000 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: min_mon_release 17 (quincy) 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: election_strategy: 1 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:40:37.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:40:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:40:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-14T03:40:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: overall HEALTH_OK 2024-09-14T03:40:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: from='mgr.25019 ' entity='' 2024-09-14T03:40:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:36 smithi110 ceph-mon[86423]: mgrmap e38: y(active, since 35s), standbys: x 2024-09-14T03:40:37.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:37.210 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: ignoring --setuser ceph since I am not root 2024-09-14T03:40:37.210 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: ignoring --setgroup ceph since I am not root 2024-09-14T03:40:37.210 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mgr[74745]: -- 172.21.15.195:0/874340281 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f02fc070e0 con 0x55f02fbe5400 2024-09-14T03:40:37.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:37.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:37.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:37.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:37.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:37.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: from='mgr.25019 172.21.15.110:0/2451011004' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:37.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:37.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: pgmap v50: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 605 KiB/s wr, 10 op/s 2024-09-14T03:40:37.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: pgmap v51: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 423 KiB/s wr, 7 op/s 2024-09-14T03:40:37.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: mon.a calling monitor election 2024-09-14T03:40:37.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:40:37.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: monmap epoch 3 2024-09-14T03:40:37.213 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:40:37.213 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: last_changed 2024-09-14T03:30:13.577071+0000 2024-09-14T03:40:37.213 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:40:37.213 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: min_mon_release 17 (quincy) 2024-09-14T03:40:37.213 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: election_strategy: 1 2024-09-14T03:40:37.213 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:40:37.214 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:40:37.214 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:40:37.214 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:40:37.214 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:40:37.214 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-14T03:40:37.215 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: overall HEALTH_OK 2024-09-14T03:40:37.215 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: from='mgr.25019 ' entity='' 2024-09-14T03:40:37.215 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:36 smithi195 ceph-mon[30695]: mgrmap e38: y(active, since 35s), standbys: x 2024-09-14T03:40:37.740 INFO:tasks.rados.rados.0.smithi110.stdout:960: done (2 left) 2024-09-14T03:40:37.740 INFO:tasks.rados.rados.0.smithi110.stdout:961: done (1 left) 2024-09-14T03:40:37.740 INFO:tasks.rados.rados.0.smithi110.stdout:962: done (0 left) 2024-09-14T03:40:37.740 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 99 2024-09-14T03:40:37.741 INFO:tasks.rados.rados.0.smithi110.stdout:963: seq_num 306 ranges {3940352=327680} 2024-09-14T03:40:37.763 INFO:tasks.rados.rados.0.smithi110.stdout:963: writing smithi11075562-15 from 3940352 to 4268032 tid 1 2024-09-14T03:40:37.763 INFO:tasks.rados.rados.0.smithi110.stdout:964: delete oid 31 current snap is 99 2024-09-14T03:40:37.766 INFO:tasks.rados.rados.0.smithi110.stdout:964: done (1 left) 2024-09-14T03:40:37.766 INFO:tasks.rados.rados.0.smithi110.stdout:965: snap_create 2024-09-14T03:40:37.769 INFO:tasks.rados.rados.0.smithi110.stdout:963: finishing write tid 1 to smithi11075562-15 2024-09-14T03:40:37.769 INFO:tasks.rados.rados.0.smithi110.stdout:963: finishing write tid 2 to smithi11075562-15 2024-09-14T03:40:37.770 INFO:tasks.rados.rados.0.smithi110.stdout:963: finishing write tid 3 to smithi11075562-15 2024-09-14T03:40:37.770 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 481 (ObjNum 306 snap 99 seq_num 306) dirty exists 2024-09-14T03:40:37.770 INFO:tasks.rados.rados.0.smithi110.stdout:963: left oid 15 (ObjNum 306 snap 99 seq_num 306) 2024-09-14T03:40:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:37 smithi110 ceph-mon[35374]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:40:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:37 smithi110 ceph-mon[86423]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:40:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:37 smithi195 ceph-mon[30695]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:40:38.658 INFO:tasks.rados.rados.0.smithi110.stdout:963: done (1 left) 2024-09-14T03:40:38.658 INFO:tasks.rados.rados.0.smithi110.stdout:965: done (0 left) 2024-09-14T03:40:38.658 INFO:tasks.rados.rados.0.smithi110.stdout:966: copy_from oid 33 from oid 44 current snap is 100 2024-09-14T03:40:38.659 INFO:tasks.rados.rados.0.smithi110.stdout:967: delete oid 3 current snap is 100 2024-09-14T03:40:38.661 INFO:tasks.rados.rados.0.smithi110.stdout:966: finishing copy_from to smithi11075562-33 2024-09-14T03:40:38.661 INFO:tasks.rados.rados.0.smithi110.stdout:966: got expected ENOENT (src dne) 2024-09-14T03:40:38.661 INFO:tasks.rados.rados.0.smithi110.stdout:966: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:40:38.662 INFO:tasks.rados.rados.0.smithi110.stdout:966: done (1 left) 2024-09-14T03:40:38.662 INFO:tasks.rados.rados.0.smithi110.stdout:967: done (0 left) 2024-09-14T03:40:38.662 INFO:tasks.rados.rados.0.smithi110.stdout:968: snap_remove snap 77 2024-09-14T03:40:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:38 smithi110 ceph-mon[35374]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:40:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:38 smithi110 ceph-mon[35374]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:40:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:38 smithi110 ceph-mon[86423]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:40:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:38 smithi110 ceph-mon[86423]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:40:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:38 smithi195 ceph-mon[30695]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:40:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:38 smithi195 ceph-mon[30695]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:40:39.661 INFO:tasks.rados.rados.0.smithi110.stdout:968: done (0 left) 2024-09-14T03:40:39.661 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 34 current snap is 100 2024-09-14T03:40:39.661 INFO:tasks.rados.rados.0.smithi110.stdout:969: seq_num 309 ranges {761856=376832} 2024-09-14T03:40:39.667 INFO:tasks.rados.rados.0.smithi110.stdout:969: writing smithi11075562-34 from 761856 to 1138688 tid 1 2024-09-14T03:40:39.668 INFO:tasks.rados.rados.0.smithi110.stdout:970: read oid 25 snap -1 2024-09-14T03:40:39.668 INFO:tasks.rados.rados.0.smithi110.stdout:970: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:40:39.668 INFO:tasks.rados.rados.0.smithi110.stdout:971: read oid 39 snap -1 2024-09-14T03:40:39.668 INFO:tasks.rados.rados.0.smithi110.stdout:971: expect deleted 2024-09-14T03:40:39.668 INFO:tasks.rados.rados.0.smithi110.stdout:972: delete oid 37 current snap is 100 2024-09-14T03:40:39.674 INFO:tasks.rados.rados.0.smithi110.stdout:969: finishing write tid 1 to smithi11075562-34 2024-09-14T03:40:39.676 INFO:tasks.rados.rados.0.smithi110.stdout:969: finishing write tid 2 to smithi11075562-34 2024-09-14T03:40:39.677 INFO:tasks.rados.rados.0.smithi110.stdout:969: finishing write tid 3 to smithi11075562-34 2024-09-14T03:40:39.677 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 309 (ObjNum 309 snap 100 seq_num 309) dirty exists 2024-09-14T03:40:39.677 INFO:tasks.rados.rados.0.smithi110.stdout:969: left oid 34 (ObjNum 309 snap 100 seq_num 309) 2024-09-14T03:40:39.679 INFO:tasks.rados.rados.0.smithi110.stdout:970: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:40:39.728 INFO:tasks.rados.rados.0.smithi110.stdout:969: done (3 left) 2024-09-14T03:40:39.728 INFO:tasks.rados.rados.0.smithi110.stdout:970: done (2 left) 2024-09-14T03:40:39.728 INFO:tasks.rados.rados.0.smithi110.stdout:971: done (1 left) 2024-09-14T03:40:39.729 INFO:tasks.rados.rados.0.smithi110.stdout:972: done (0 left) 2024-09-14T03:40:39.729 INFO:tasks.rados.rados.0.smithi110.stdout:973: read oid 31 snap -1 2024-09-14T03:40:39.729 INFO:tasks.rados.rados.0.smithi110.stdout:973: expect deleted 2024-09-14T03:40:39.729 INFO:tasks.rados.rados.0.smithi110.stdout:974: delete oid 48 current snap is 100 2024-09-14T03:40:39.729 INFO:tasks.rados.rados.0.smithi110.stdout:973: done (1 left) 2024-09-14T03:40:39.730 INFO:tasks.rados.rados.0.smithi110.stdout:974: done (0 left) 2024-09-14T03:40:39.730 INFO:tasks.rados.rados.0.smithi110.stdout:975: read oid 25 snap -1 2024-09-14T03:40:39.730 INFO:tasks.rados.rados.0.smithi110.stdout:975: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:40:39.730 INFO:tasks.rados.rados.0.smithi110.stdout:976: snap_remove snap 97 2024-09-14T03:40:40.657 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:40.665 INFO:tasks.rados.rados.0.smithi110.stdout:976: done (1 left) 2024-09-14T03:40:40.665 INFO:tasks.rados.rados.0.smithi110.stdout:977: snap_remove snap 88 2024-09-14T03:40:40.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:40 smithi110 ceph-mon[35374]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:40:40.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:40 smithi110 ceph-mon[86423]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:40:40.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:40 smithi195 ceph-mon[30695]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:40:41.669 INFO:tasks.rados.rados.0.smithi110.stdout:977: done (1 left) 2024-09-14T03:40:41.669 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 100 2024-09-14T03:40:41.669 INFO:tasks.rados.rados.0.smithi110.stdout:978: seq_num 312 ranges {3538944=434176} 2024-09-14T03:40:41.691 INFO:tasks.rados.rados.0.smithi110.stdout:978: writing smithi11075562-14 from 3538944 to 3973120 tid 1 2024-09-14T03:40:41.691 INFO:tasks.rados.rados.0.smithi110.stdout:979: setattr oid 13 current snap is 100 2024-09-14T03:40:41.692 INFO:tasks.rados.rados.0.smithi110.stdout:975: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:40:41.765 INFO:tasks.rados.rados.0.smithi110.stdout:978: finishing write tid 1 to smithi11075562-14 2024-09-14T03:40:41.765 INFO:tasks.rados.rados.0.smithi110.stdout:978: finishing write tid 2 to smithi11075562-14 2024-09-14T03:40:41.765 INFO:tasks.rados.rados.0.smithi110.stdout:978: finishing write tid 3 to smithi11075562-14 2024-09-14T03:40:41.765 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 534 (ObjNum 312 snap 100 seq_num 312) dirty exists 2024-09-14T03:40:41.765 INFO:tasks.rados.rados.0.smithi110.stdout:978: left oid 14 (ObjNum 312 snap 100 seq_num 312) 2024-09-14T03:40:41.765 INFO:tasks.rados.rados.0.smithi110.stdout:975: done (2 left) 2024-09-14T03:40:41.765 INFO:tasks.rados.rados.0.smithi110.stdout:978: done (1 left) 2024-09-14T03:40:41.765 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 100 2024-09-14T03:40:41.766 INFO:tasks.rados.rados.0.smithi110.stdout:980: seq_num 313 ranges {3670016=450560} 2024-09-14T03:40:41.787 INFO:tasks.rados.rados.0.smithi110.stdout:980: writing smithi11075562-24 from 3670016 to 4120576 tid 1 2024-09-14T03:40:41.787 INFO:tasks.rados.rados.0.smithi110.stdout:981: rollback oid 4 current snap is 100 2024-09-14T03:40:41.787 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 4 to 78 2024-09-14T03:40:41.787 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 571 (ObjNum 287 snap 86 seq_num 287) dirty exists 2024-09-14T03:40:41.787 INFO:tasks.rados.rados.0.smithi110.stdout:979: done (2 left) 2024-09-14T03:40:41.787 INFO:tasks.rados.rados.0.smithi110.stdout:982: snap_remove snap 79 2024-09-14T03:40:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:41 smithi110 ceph-mon[35374]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:40:41.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:41 smithi110 ceph-mon[86423]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:40:41.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:41 smithi195 ceph-mon[30695]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:40:41.972 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:41 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: [14/Sep/2024:03:40:41] ENGINE Bus STARTING 2024-09-14T03:40:41.972 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:41 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: CherryPy Checker: 2024-09-14T03:40:41.972 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:41 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: The Application mounted at '' has an empty config. 2024-09-14T03:40:41.972 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:41 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: 2024-09-14T03:40:41.972 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:41 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: [14/Sep/2024:03:40:41] ENGINE Serving on http://:::9283 2024-09-14T03:40:41.972 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:40:41 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x[74741]: [14/Sep/2024:03:40:41] ENGINE Bus STARTED 2024-09-14T03:40:42.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.081+0000 7f93dbd58640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.081+0000 7f93dbd58640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.081+0000 7f93dbd58640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.081+0000 7f93dbd58640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.081+0000 7f93dbd58640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.082+0000 7f93df55f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.082+0000 7f93df55f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.082+0000 7f93df55f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.082+0000 7f93df55f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.082+0000 7f93df55f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:40:42.720 INFO:tasks.rados.rados.0.smithi110.stdout:982: done (2 left) 2024-09-14T03:40:42.720 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 100 2024-09-14T03:40:42.720 INFO:tasks.rados.rados.0.smithi110.stdout:983: seq_num 314 ranges {0=466944} 2024-09-14T03:40:42.721 INFO:tasks.rados.rados.0.smithi110.stdout:983: writing smithi11075562-17 from 0 to 466944 tid 1 2024-09-14T03:40:42.721 INFO:tasks.rados.rados.0.smithi110.stdout:984: read oid 2 snap -1 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:984: expect (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:985: setattr oid 29 current snap is 100 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:981: finishing rollback tid 0 to smithi11075562-4 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:981: finishing rollback tid 1 to smithi11075562-4 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:980: finishing write tid 1 to smithi11075562-24 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:980: finishing write tid 2 to smithi11075562-24 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:981: finishing rollback tid 2 to smithi11075562-4 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 312 (ObjNum 255 snap 76 seq_num 255) dirty exists 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:980: finishing write tid 3 to smithi11075562-24 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 268 (ObjNum 313 snap 100 seq_num 313) dirty exists 2024-09-14T03:40:42.722 INFO:tasks.rados.rados.0.smithi110.stdout:980: left oid 24 (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:40:42.725 INFO:tasks.rados.rados.0.smithi110.stdout:980: done (4 left) 2024-09-14T03:40:42.725 INFO:tasks.rados.rados.0.smithi110.stdout:981: done (3 left) 2024-09-14T03:40:42.725 INFO:tasks.rados.rados.0.smithi110.stdout:986: delete oid 12 current snap is 100 2024-09-14T03:40:42.731 INFO:tasks.rados.rados.0.smithi110.stdout:986: done (3 left) 2024-09-14T03:40:42.731 INFO:tasks.rados.rados.0.smithi110.stdout:987: rollback oid 48 current snap is 100 2024-09-14T03:40:42.731 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 48 to 78 2024-09-14T03:40:42.731 INFO:tasks.rados.rados.0.smithi110.stdout:988: rollback oid 32 current snap is 100 2024-09-14T03:40:42.731 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 98 2024-09-14T03:40:42.731 INFO:tasks.rados.rados.0.smithi110.stdout:989: rmattr oid 30 current snap is 100 2024-09-14T03:40:42.731 INFO:tasks.rados.rados.0.smithi110.stdout:990: read oid 43 snap -1 2024-09-14T03:40:42.732 INFO:tasks.rados.rados.0.smithi110.stdout:990: expect deleted 2024-09-14T03:40:42.732 INFO:tasks.rados.rados.0.smithi110.stdout:991: snap_create 2024-09-14T03:40:42.733 INFO:tasks.rados.rados.0.smithi110.stdout:983: finishing write tid 1 to smithi11075562-17 2024-09-14T03:40:42.733 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 268 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-14T03:40:42.734 INFO:tasks.rados.rados.0.smithi110.stdout:983: finishing write tid 2 to smithi11075562-17 2024-09-14T03:40:42.737 INFO:tasks.rados.rados.0.smithi110.stdout:987: finishing rollback tid 1 to smithi11075562-48 2024-09-14T03:40:42.741 INFO:tasks.rados.rados.0.smithi110.stdout:987: finishing rollback tid 2 to smithi11075562-48 2024-09-14T03:40:42.741 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 395 (ObjNum 214 snap 65 seq_num 214) dirty exists 2024-09-14T03:40:42.741 INFO:tasks.rados.rados.0.smithi110.stdout:983: finishing write tid 3 to smithi11075562-17 2024-09-14T03:40:42.741 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 393 (ObjNum 314 snap 100 seq_num 314) dirty exists 2024-09-14T03:40:42.741 INFO:tasks.rados.rados.0.smithi110.stdout:983: left oid 17 (ObjNum 314 snap 100 seq_num 314) 2024-09-14T03:40:42.744 INFO:tasks.rados.rados.0.smithi110.stdout:988: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:40:42.744 INFO:tasks.rados.rados.0.smithi110.stdout:988: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:40:42.744 INFO:tasks.rados.rados.0.smithi110.stdout:988: finishing rollback tid 2 to smithi11075562-32 2024-09-14T03:40:42.744 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 494 (ObjNum 235 snap 69 seq_num 235) dirty exists 2024-09-14T03:40:42.744 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 496 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:40:42.744 INFO:tasks.rados.rados.0.smithi110.stdout:984: expect (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:40:42.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.703+0000 7f940e88b640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T03:40:42.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:42] ENGINE Bus STARTING 2024-09-14T03:40:42.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: CherryPy Checker: 2024-09-14T03:40:42.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: The Application mounted at '' has an empty config. 2024-09-14T03:40:42.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:42.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:42] ENGINE Serving on http://:::9283 2024-09-14T03:40:42.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:42] ENGINE Bus STARTED 2024-09-14T03:40:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:40:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: Standby manager daemon x restarted 2024-09-14T03:40:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: Standby manager daemon x started 2024-09-14T03:40:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: Active manager daemon y restarted 2024-09-14T03:40:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: Activating manager daemon y 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: mgrmap e39: y(active, starting, since 0.010548s), standbys: x 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:40:42.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:42.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:42.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:42.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:40:42.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: Manager daemon y is now available 2024-09-14T03:40:42.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:42.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:42.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:42.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:40:42.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:40:42.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:40:42.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:40:42.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: Standby manager daemon x restarted 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: Standby manager daemon x started 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: Active manager daemon y restarted 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: Activating manager daemon y 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: mgrmap e39: y(active, starting, since 0.010548s), standbys: x 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:42.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:42.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:40:42.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:40:42.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:40:42.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:42.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:42.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:42.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:40:42.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:40:42.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:40:42.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:42.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:42.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: Manager daemon y is now available 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:40:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:42 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: Standby manager daemon x restarted 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: Standby manager daemon x started 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: Active manager daemon y restarted 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: Activating manager daemon y 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.? 172.21.15.195:0/2527690631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: mgrmap e39: y(active, starting, since 0.010548s), standbys: x 2024-09-14T03:40:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:40:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:40:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:40:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: Manager daemon y is now available 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:40:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:42 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:43.178+0000 7f93bcb60640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:40:43] ENGINE Error in HTTPServer.serve 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: Traceback (most recent call last): 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: self._connections.run(self.expiration_interval) 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: self._run(expiration_interval) 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:40:43.447 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: s = self.context.wrap_socket( 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: return self.sslsocket_class._create( 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: self.do_handshake() 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: self._sslobj.do_handshake() 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:40:43.448 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:40:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:983: done (7 left) 2024-09-14T03:40:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:984: done (6 left) 2024-09-14T03:40:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:985: done (5 left) 2024-09-14T03:40:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:987: done (4 left) 2024-09-14T03:40:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:988: done (3 left) 2024-09-14T03:40:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:989: done (2 left) 2024-09-14T03:40:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:990: done (1 left) 2024-09-14T03:40:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:991: done (0 left) 2024-09-14T03:40:43.718 INFO:tasks.rados.rados.0.smithi110.stdout:992: snap_remove snap 85 2024-09-14T03:40:43.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:43 smithi195 ceph-mon[30695]: mgrmap e40: y(active, since 1.01035s), standbys: x 2024-09-14T03:40:43.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:43 smithi195 ceph-mon[30695]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:40:43.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:43 smithi195 ceph-mon[30695]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:43.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:43 smithi195 ceph-mon[30695]: pgmap v4: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 48 GiB/s rd, 20 GiB/s wr, 201.43k op/s 2024-09-14T03:40:44.004 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-mon[35374]: mgrmap e40: y(active, since 1.01035s), standbys: x 2024-09-14T03:40:44.004 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-mon[35374]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:40:44.005 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-mon[35374]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:44.005 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-mon[35374]: pgmap v4: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 48 GiB/s rd, 20 GiB/s wr, 201.43k op/s 2024-09-14T03:40:44.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-mon[86423]: mgrmap e40: y(active, since 1.01035s), standbys: x 2024-09-14T03:40:44.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-mon[86423]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:40:44.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-mon[86423]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:40:44.005 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:43 smithi110 ceph-mon[86423]: pgmap v4: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 48 GiB/s rd, 20 GiB/s wr, 201.43k op/s 2024-09-14T03:40:44.721 INFO:tasks.rados.rados.0.smithi110.stdout:992: done (0 left) 2024-09-14T03:40:44.722 INFO:tasks.rados.rados.0.smithi110.stdout:993: read oid 47 snap -1 2024-09-14T03:40:44.722 INFO:tasks.rados.rados.0.smithi110.stdout:993: expect (ObjNum 305 snap 98 seq_num 305) 2024-09-14T03:40:44.722 INFO:tasks.rados.rados.0.smithi110.stdout:994: read oid 8 snap -1 2024-09-14T03:40:44.722 INFO:tasks.rados.rados.0.smithi110.stdout:994: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:40:44.722 INFO:tasks.rados.rados.0.smithi110.stdout:995: snap_create 2024-09-14T03:40:44.736 INFO:tasks.rados.rados.0.smithi110.stdout:994: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:40:44.783 INFO:tasks.rados.rados.0.smithi110.stdout:993: expect (ObjNum 305 snap 98 seq_num 305) 2024-09-14T03:40:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:42] ENGINE Bus STARTING 2024-09-14T03:40:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:43] ENGINE Serving on http://172.21.15.110:8765 2024-09-14T03:40:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:43] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:40:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:43] ENGINE Bus STARTED 2024-09-14T03:40:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[35374]: [14/Sep/2024:03:40:43] ENGINE Error in HTTPServer.serve 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:40:45.198 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.c.smithi110.stdout: self.do_handshake() 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.c.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.c.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.c.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[35374]: pgmap v5: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 523 MiB/s rd, 214 MiB/s wr, 2.15k op/s 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[35374]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[35374]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[86423]: [14/Sep/2024:03:40:42] ENGINE Bus STARTING 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[86423]: [14/Sep/2024:03:40:43] ENGINE Serving on http://172.21.15.110:8765 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[86423]: [14/Sep/2024:03:40:43] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:40:45.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[86423]: [14/Sep/2024:03:40:43] ENGINE Bus STARTED 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[86423]: [14/Sep/2024:03:40:43] ENGINE Error in HTTPServer.serve 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: Traceback (most recent call last): 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._run(expiration_interval) 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:40:45.200 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout: s = self.context.wrap_socket( 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout: return self.sslsocket_class._create( 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout: self.do_handshake() 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout: self._sslobj.do_handshake() 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[86423]: pgmap v5: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 523 MiB/s rd, 214 MiB/s wr, 2.15k op/s 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[86423]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:40:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:44 smithi110 ceph-mon[86423]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:44 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:42] ENGINE Bus STARTING 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:44 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:43] ENGINE Serving on http://172.21.15.110:8765 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:44 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:43] ENGINE Serving on https://172.21.15.110:7150 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:44 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:43] ENGINE Bus STARTED 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:44 smithi195 ceph-mon[30695]: [14/Sep/2024:03:40:43] ENGINE Error in HTTPServer.serve 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: Traceback (most recent call last): 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._run(expiration_interval) 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:40:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: s = self.context.wrap_socket( 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: return self.sslsocket_class._create( 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: self.do_handshake() 2024-09-14T03:40:45.210 INFO:journalctl@ceph.mon.b.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:40:45.211 INFO:journalctl@ceph.mon.b.smithi195.stdout: self._sslobj.do_handshake() 2024-09-14T03:40:45.211 INFO:journalctl@ceph.mon.b.smithi195.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:40:45.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:44 smithi195 ceph-mon[30695]: pgmap v5: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 523 MiB/s rd, 214 MiB/s wr, 2.15k op/s 2024-09-14T03:40:45.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:44 smithi195 ceph-mon[30695]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:40:45.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:44 smithi195 ceph-mon[30695]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-14T03:40:45.738 INFO:tasks.rados.rados.0.smithi110.stdout:993: done (2 left) 2024-09-14T03:40:45.738 INFO:tasks.rados.rados.0.smithi110.stdout:994: done (1 left) 2024-09-14T03:40:45.739 INFO:tasks.rados.rados.0.smithi110.stdout:995: done (0 left) 2024-09-14T03:40:45.739 INFO:tasks.rados.rados.0.smithi110.stdout:996: rollback oid 44 current snap is 102 2024-09-14T03:40:45.739 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 44 to 90 2024-09-14T03:40:45.739 INFO:tasks.rados.rados.0.smithi110.stdout:997: copy_from oid 18 from oid 38 current snap is 102 2024-09-14T03:40:45.739 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 102 2024-09-14T03:40:45.739 INFO:tasks.rados.rados.0.smithi110.stdout:998: seq_num 316 ranges {0=499712} 2024-09-14T03:40:45.741 INFO:tasks.rados.rados.0.smithi110.stdout:998: writing smithi11075562-3 from 0 to 499712 tid 1 2024-09-14T03:40:45.741 INFO:tasks.rados.rados.0.smithi110.stdout:999: snap_remove snap 101 2024-09-14T03:40:46.092 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:45 smithi195 ceph-mon[30695]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:40:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:45 smithi110 ceph-mon[35374]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:40:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:45 smithi110 ceph-mon[86423]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:40:46.278 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:999: done (3 left) 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:1000: snap_create 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:996: finishing rollback tid 1 to smithi11075562-44 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 376 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:998: finishing write tid 1 to smithi11075562-3 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:998: finishing write tid 2 to smithi11075562-3 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:998: finishing write tid 3 to smithi11075562-3 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 369 (ObjNum 316 snap 102 seq_num 316) dirty exists 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:998: left oid 3 (ObjNum 316 snap 102 seq_num 316) 2024-09-14T03:40:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:997: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:40:46.736 INFO:tasks.rados.rados.0.smithi110.stdout:997: finishing copy_from to smithi11075562-18 2024-09-14T03:40:46.736 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 241 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:40:46.862 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:40:46.862 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (87s) 0s ago 5m 16.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:40:46.862 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (64s) 0s ago 5m 77.5M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:40:46.862 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (7m) 0s ago 6m 18.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (6m) 0s ago 6m 16.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (27s) 0s ago 10m 451M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (54s) 0s ago 11m 488M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (13s) 0s ago 12m 31.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (10m) 0s ago 10m 56.4M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 b6415a0de807 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (10m) 0s ago 10m 52.8M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 8e2894672cd8 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (82s) 0s ago 5m 9.91M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (59s) 0s ago 5m 7936k - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (10m) 0s ago 10m 158M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (9m) 0s ago 9m 153M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (9m) 0s ago 9m 122M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (8m) 0s ago 8m 113M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:40:46.863 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (8m) 0s ago 8m 206M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:40:46.864 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (8m) 0s ago 8m 152M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:40:46.864 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (7m) 0s ago 7m 105M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:40:46.864 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (7m) 0s ago 7m 98.8M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:40:46.864 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (32s) 0s ago 6m 41.9M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:40:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: pgmap v8: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 174 MiB/s rd, 71 MiB/s wr, 715 op/s 2024-09-14T03:40:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-14T03:40:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:40:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: pgmap v8: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 174 MiB/s rd, 71 MiB/s wr, 715 op/s 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:46 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2, 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:40:47.207 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:40:47.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:40:47.208 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:40:47.208 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 12, 2024-09-14T03:40:47.208 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:40:47.208 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:40:47.208 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:40:47.208 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: pgmap v8: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 262 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 174 MiB/s rd, 71 MiB/s wr, 715 op/s 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:46 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.489 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr" 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "3/20 daemons upgraded", 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "", 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:40:47.490 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:40:47.740 INFO:tasks.rados.rados.0.smithi110.stdout:996: done (3 left) 2024-09-14T03:40:47.740 INFO:tasks.rados.rados.0.smithi110.stdout:997: done (2 left) 2024-09-14T03:40:47.740 INFO:tasks.rados.rados.0.smithi110.stdout:998: done (1 left) 2024-09-14T03:40:47.741 INFO:tasks.rados.rados.0.smithi110.stdout:1000: done (0 left) 2024-09-14T03:40:47.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 103 2024-09-14T03:40:47.741 INFO:tasks.rados.rados.0.smithi110.stdout:1001: seq_num 317 ranges {3162112=516096} 2024-09-14T03:40:47.760 INFO:tasks.rados.rados.0.smithi110.stdout:1001: writing smithi11075562-1 from 3162112 to 3678208 tid 1 2024-09-14T03:40:47.760 INFO:tasks.rados.rados.0.smithi110.stdout:1002: read oid 35 snap -1 2024-09-14T03:40:47.760 INFO:tasks.rados.rados.0.smithi110.stdout:1002: expect deleted 2024-09-14T03:40:47.760 INFO:tasks.rados.rados.0.smithi110.stdout:1003: snap_create 2024-09-14T03:40:47.765 INFO:tasks.rados.rados.0.smithi110.stdout:1001: finishing write tid 1 to smithi11075562-1 2024-09-14T03:40:47.766 INFO:tasks.rados.rados.0.smithi110.stdout:1001: finishing write tid 2 to smithi11075562-1 2024-09-14T03:40:47.767 INFO:tasks.rados.rados.0.smithi110.stdout:1001: finishing write tid 3 to smithi11075562-1 2024-09-14T03:40:47.767 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 588 (ObjNum 317 snap 103 seq_num 317) dirty exists 2024-09-14T03:40:47.767 INFO:tasks.rados.rados.0.smithi110.stdout:1001: left oid 1 (ObjNum 317 snap 103 seq_num 317) 2024-09-14T03:40:47.826 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:47 smithi195 ceph-mon[30695]: from='client.25157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:47.826 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:47 smithi195 ceph-mon[30695]: from='client.25163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:47.826 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:47 smithi195 ceph-mon[30695]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:40:47.826 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:47 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.827 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:47 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:47.827 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:47 smithi195 ceph-mon[30695]: from='client.? 172.21.15.110:0/441801125' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:40:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[35374]: from='client.25157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[35374]: from='client.25163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[35374]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:40:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[35374]: from='client.? 172.21.15.110:0/441801125' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:40:48.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[86423]: from='client.25157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:48.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[86423]: from='client.25163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:48.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[86423]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:40:48.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:48.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:48.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:47 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/441801125' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:40:48.663 INFO:tasks.rados.rados.0.smithi110.stdout:1001: done (2 left) 2024-09-14T03:40:48.664 INFO:tasks.rados.rados.0.smithi110.stdout:1002: done (1 left) 2024-09-14T03:40:48.664 INFO:tasks.rados.rados.0.smithi110.stdout:1003: done (0 left) 2024-09-14T03:40:48.664 INFO:tasks.rados.rados.0.smithi110.stdout:1004: rollback oid 37 current snap is 104 2024-09-14T03:40:48.664 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 37 to 66 2024-09-14T03:40:48.664 INFO:tasks.rados.rados.0.smithi110.stdout:1005: delete oid 44 current snap is 104 2024-09-14T03:40:48.669 INFO:tasks.rados.rados.0.smithi110.stdout:1005: done (1 left) 2024-09-14T03:40:48.669 INFO:tasks.rados.rados.0.smithi110.stdout:1006: copy_from oid 39 from oid 8 current snap is 104 2024-09-14T03:40:48.669 INFO:tasks.rados.rados.0.smithi110.stdout:1007: read oid 46 snap 87 2024-09-14T03:40:48.670 INFO:tasks.rados.rados.0.smithi110.stdout:1007: expect deleted 2024-09-14T03:40:48.670 INFO:tasks.rados.rados.0.smithi110.stdout:1008: rollback oid 29 current snap is 104 2024-09-14T03:40:48.670 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 29 to 102 2024-09-14T03:40:48.670 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 104 2024-09-14T03:40:48.670 INFO:tasks.rados.rados.0.smithi110.stdout:1009: seq_num 319 ranges {0=548864} 2024-09-14T03:40:48.673 INFO:tasks.rados.rados.0.smithi110.stdout:1009: writing smithi11075562-50 from 0 to 548864 tid 1 2024-09-14T03:40:48.673 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 104 2024-09-14T03:40:48.673 INFO:tasks.rados.rados.0.smithi110.stdout:1010: seq_num 320 ranges {4120576=565248} 2024-09-14T03:40:48.698 INFO:tasks.rados.rados.0.smithi110.stdout:1010: writing smithi11075562-24 from 4120576 to 4685824 tid 1 2024-09-14T03:40:48.699 INFO:tasks.rados.rados.0.smithi110.stdout:1011: snap_remove snap 86 2024-09-14T03:40:48.903 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:48 smithi195 ceph-mon[30695]: from='client.25169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:48.903 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:48 smithi195 ceph-mon[30695]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:48.903 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:48 smithi195 ceph-mon[30695]: pgmap v11: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 261 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 122 KiB/s wr, 20 op/s 2024-09-14T03:40:48.904 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:48 smithi195 ceph-mon[30695]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:40:48.904 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:48 smithi195 ceph-mon[30695]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:40:49.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[35374]: from='client.25169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:49.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[35374]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:49.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[35374]: pgmap v11: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 261 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 122 KiB/s wr, 20 op/s 2024-09-14T03:40:49.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[35374]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:40:49.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[35374]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:40:49.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[86423]: from='client.25169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:49.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[86423]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:49.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[86423]: pgmap v11: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 261 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 122 KiB/s wr, 20 op/s 2024-09-14T03:40:49.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[86423]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:40:49.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:48 smithi110 ceph-mon[86423]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:1004: finishing rollback tid 1 to smithi11075562-37 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 593 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:1008: finishing rollback tid 0 to smithi11075562-29 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:1008: finishing rollback tid 1 to smithi11075562-29 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:1008: finishing rollback tid 2 to smithi11075562-29 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 274 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:1009: finishing write tid 1 to smithi11075562-50 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:1009: finishing write tid 2 to smithi11075562-50 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:1006: finishing copy_from racing read to smithi11075562-39 2024-09-14T03:40:49.667 INFO:tasks.rados.rados.0.smithi110.stdout:1009: finishing write tid 3 to smithi11075562-50 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 90 (ObjNum 319 snap 104 seq_num 319) dirty exists 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1009: left oid 50 (ObjNum 319 snap 104 seq_num 319) 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1006: finishing copy_from to smithi11075562-39 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 472 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1010: finishing write tid 1 to smithi11075562-24 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1010: finishing write tid 2 to smithi11075562-24 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1010: finishing write tid 3 to smithi11075562-24 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 270 (ObjNum 320 snap 104 seq_num 320) dirty exists 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1010: left oid 24 (ObjNum 320 snap 104 seq_num 320) 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1004: done (6 left) 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1006: done (5 left) 2024-09-14T03:40:49.668 INFO:tasks.rados.rados.0.smithi110.stdout:1007: done (4 left) 2024-09-14T03:40:49.669 INFO:tasks.rados.rados.0.smithi110.stdout:1008: done (3 left) 2024-09-14T03:40:49.669 INFO:tasks.rados.rados.0.smithi110.stdout:1009: done (2 left) 2024-09-14T03:40:49.669 INFO:tasks.rados.rados.0.smithi110.stdout:1010: done (1 left) 2024-09-14T03:40:49.669 INFO:tasks.rados.rados.0.smithi110.stdout:1011: done (0 left) 2024-09-14T03:40:49.669 INFO:tasks.rados.rados.0.smithi110.stdout:1012: rmattr oid 25 current snap is 104 2024-09-14T03:40:49.669 INFO:tasks.rados.rados.0.smithi110.stdout:1012: done (0 left) 2024-09-14T03:40:49.669 INFO:tasks.rados.rados.0.smithi110.stdout:1013: setattr oid 27 current snap is 104 2024-09-14T03:40:49.669 INFO:tasks.rados.rados.0.smithi110.stdout:1014: snap_remove snap 81 2024-09-14T03:40:50.665 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:50.671 INFO:tasks.rados.rados.0.smithi110.stdout:1014: done (1 left) 2024-09-14T03:40:50.671 INFO:tasks.rados.rados.0.smithi110.stdout:1015: rollback oid 16 current snap is 104 2024-09-14T03:40:50.671 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 82 2024-09-14T03:40:50.671 INFO:tasks.rados.rados.0.smithi110.stdout:1016: snap_remove snap 93 2024-09-14T03:40:50.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[35374]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:40:50.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[35374]: pgmap v15: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.8 MiB/s wr, 29 op/s 2024-09-14T03:40:50.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:50.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:50.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:50.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:50.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[86423]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:40:50.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[86423]: pgmap v15: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.8 MiB/s wr, 29 op/s 2024-09-14T03:40:50.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:50.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:50.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:50.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:50 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:50 smithi195 ceph-mon[30695]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:40:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:50 smithi195 ceph-mon[30695]: pgmap v15: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.8 MiB/s wr, 29 op/s 2024-09-14T03:40:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:50 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:50 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:50 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:50 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:51.674 INFO:tasks.rados.rados.0.smithi110.stdout:1016: done (2 left) 2024-09-14T03:40:51.674 INFO:tasks.rados.rados.0.smithi110.stdout:1017: read oid 28 snap -1 2024-09-14T03:40:51.675 INFO:tasks.rados.rados.0.smithi110.stdout:1017: expect (ObjNum 302 snap 97 seq_num 302) 2024-09-14T03:40:51.675 INFO:tasks.rados.rados.0.smithi110.stdout:1018: copy_from oid 15 from oid 9 current snap is 104 2024-09-14T03:40:51.675 INFO:tasks.rados.rados.0.smithi110.stdout:1019: snap_remove snap 25 2024-09-14T03:40:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:40:51] "GET /metrics HTTP/1.1" 200 33394 "" "Prometheus/2.51.0" 2024-09-14T03:40:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:40:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:40:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:51.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:51.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:51.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:40:51.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:51 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:40:51.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-14T03:40:51.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-14T03:40:51.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: Updating smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: Updating smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:51.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:51.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:40:51.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:40:51.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:51 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:40:52.681 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: pgmap v18: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.684 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.684 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:52.684 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:52.684 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:52.684 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:52 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:52.684 INFO:tasks.rados.rados.0.smithi110.stdout:1019: done (4 left) 2024-09-14T03:40:52.684 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 104 2024-09-14T03:40:52.685 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 99 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:40:52.685 INFO:tasks.rados.rados.0.smithi110.stdout:1015: finishing rollback tid 0 to smithi11075562-16 2024-09-14T03:40:52.685 INFO:tasks.rados.rados.0.smithi110.stdout:1015: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:40:52.685 INFO:tasks.rados.rados.0.smithi110.stdout:1015: finishing rollback tid 2 to smithi11075562-16 2024-09-14T03:40:52.685 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 457 (ObjNum 234 snap 69 seq_num 234) dirty exists 2024-09-14T03:40:52.685 INFO:tasks.rados.rados.0.smithi110.stdout:1017: expect (ObjNum 302 snap 97 seq_num 302) 2024-09-14T03:40:52.864 INFO:tasks.rados.rados.0.smithi110.stdout:1018: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:40:52.864 INFO:tasks.rados.rados.0.smithi110.stdout:1020: seq_num 321 ranges {0=581632} 2024-09-14T03:40:52.867 INFO:tasks.rados.rados.0.smithi110.stdout:1020: writing smithi11075562-46 from 0 to 581632 tid 1 2024-09-14T03:40:52.867 INFO:tasks.rados.rados.0.smithi110.stdout:1013: done (4 left) 2024-09-14T03:40:52.867 INFO:tasks.rados.rados.0.smithi110.stdout:1015: done (3 left) 2024-09-14T03:40:52.867 INFO:tasks.rados.rados.0.smithi110.stdout:1017: done (2 left) 2024-09-14T03:40:52.867 INFO:tasks.rados.rados.0.smithi110.stdout:1021: delete oid 25 current snap is 104 2024-09-14T03:40:52.867 INFO:tasks.rados.rados.0.smithi110.stdout:1018: finishing copy_from to smithi11075562-15 2024-09-14T03:40:52.868 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 309 (ObjNum 271 snap 81 seq_num 271) dirty exists 2024-09-14T03:40:52.869 INFO:tasks.rados.rados.0.smithi110.stdout:1018: done (2 left) 2024-09-14T03:40:52.869 INFO:tasks.rados.rados.0.smithi110.stdout:1021: done (1 left) 2024-09-14T03:40:52.869 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 104 2024-09-14T03:40:52.869 INFO:tasks.rados.rados.0.smithi110.stdout:1022: seq_num 323 ranges {1548288=614400} 2024-09-14T03:40:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:1022: writing smithi11075562-19 from 1548288 to 2162688 tid 1 2024-09-14T03:40:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:1023: rollback oid 35 current snap is 104 2024-09-14T03:40:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 96 2024-09-14T03:40:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:1024: snap_create 2024-09-14T03:40:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:1020: finishing write tid 1 to smithi11075562-46 2024-09-14T03:40:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:1020: finishing write tid 2 to smithi11075562-46 2024-09-14T03:40:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:1020: finishing write tid 3 to smithi11075562-46 2024-09-14T03:40:52.882 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 477 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:40:52.882 INFO:tasks.rados.rados.0.smithi110.stdout:1020: left oid 46 (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:40:52.883 INFO:tasks.rados.rados.0.smithi110.stdout:1023: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:40:52.883 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 470 (ObjNum 177 snap 0 seq_num 0) dirty dne 2024-09-14T03:40:52.885 INFO:tasks.rados.rados.0.smithi110.stdout:1022: finishing write tid 1 to smithi11075562-19 2024-09-14T03:40:52.885 INFO:tasks.rados.rados.0.smithi110.stdout:1022: finishing write tid 2 to smithi11075562-19 2024-09-14T03:40:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:1022: finishing write tid 3 to smithi11075562-19 2024-09-14T03:40:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 474 (ObjNum 323 snap 104 seq_num 323) dirty exists 2024-09-14T03:40:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:1022: left oid 19 (ObjNum 323 snap 104 seq_num 323) 2024-09-14T03:40:52.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:40:52.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:40:52.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: pgmap v18: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:40:52.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: pgmap v18: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:52.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:52.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:52.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[35374]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:52.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:52.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:40:52.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:52.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:52.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:52 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:53.684 INFO:tasks.rados.rados.0.smithi110.stdout:1020: done (3 left) 2024-09-14T03:40:53.684 INFO:tasks.rados.rados.0.smithi110.stdout:1022: done (2 left) 2024-09-14T03:40:53.684 INFO:tasks.rados.rados.0.smithi110.stdout:1023: done (1 left) 2024-09-14T03:40:53.684 INFO:tasks.rados.rados.0.smithi110.stdout:1024: done (0 left) 2024-09-14T03:40:53.684 INFO:tasks.rados.rados.0.smithi110.stdout:1025: rmattr oid 32 current snap is 105 2024-09-14T03:40:53.684 INFO:tasks.rados.rados.0.smithi110.stdout:1025: done (0 left) 2024-09-14T03:40:53.685 INFO:tasks.rados.rados.0.smithi110.stdout:1026: read oid 50 snap -1 2024-09-14T03:40:53.685 INFO:tasks.rados.rados.0.smithi110.stdout:1026: expect (ObjNum 319 snap 104 seq_num 319) 2024-09-14T03:40:53.685 INFO:tasks.rados.rados.0.smithi110.stdout:1027: rollback oid 48 current snap is 105 2024-09-14T03:40:53.685 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 48 to 96 2024-09-14T03:40:53.685 INFO:tasks.rados.rados.0.smithi110.stdout:1028: delete oid 5 current snap is 105 2024-09-14T03:40:53.688 INFO:tasks.rados.rados.0.smithi110.stdout:1026: expect (ObjNum 319 snap 104 seq_num 319) 2024-09-14T03:40:53.703 INFO:tasks.rados.rados.0.smithi110.stdout:1027: finishing rollback tid 0 to smithi11075562-48 2024-09-14T03:40:53.703 INFO:tasks.rados.rados.0.smithi110.stdout:1027: finishing rollback tid 1 to smithi11075562-48 2024-09-14T03:40:53.703 INFO:tasks.rados.rados.0.smithi110.stdout:1026: done (2 left) 2024-09-14T03:40:53.704 INFO:tasks.rados.rados.0.smithi110.stdout:1028: done (1 left) 2024-09-14T03:40:53.704 INFO:tasks.rados.rados.0.smithi110.stdout:1029: delete oid 45 current snap is 105 2024-09-14T03:40:53.704 INFO:tasks.rados.rados.0.smithi110.stdout:1027: finishing rollback tid 2 to smithi11075562-48 2024-09-14T03:40:53.704 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 398 (ObjNum 299 snap 96 seq_num 299) dirty exists 2024-09-14T03:40:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1027: done (1 left) 2024-09-14T03:40:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1029: done (0 left) 2024-09-14T03:40:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1030: copy_from oid 25 from oid 31 current snap is 105 2024-09-14T03:40:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1031: read oid 49 snap 99 2024-09-14T03:40:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1031: expect deleted 2024-09-14T03:40:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 105 2024-09-14T03:40:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1032: seq_num 326 ranges {4153344=663552} 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:1032: writing smithi11075562-23 from 4153344 to 4816896 tid 1 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:1033: copy_from oid 32 from oid 42 current snap is 105 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:1030: finishing copy_from to smithi11075562-25 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:1030: got expected ENOENT (src dne) 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:1030: finishing copy_from racing read to smithi11075562-25 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:1030: done (3 left) 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:1031: done (2 left) 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 105 2024-09-14T03:40:53.733 INFO:tasks.rados.rados.0.smithi110.stdout:1034: seq_num 327 ranges {6545408=679936} 2024-09-14T03:40:53.772 INFO:tasks.rados.rados.0.smithi110.stdout:1034: writing smithi11075562-10 from 6545408 to 7225344 tid 1 2024-09-14T03:40:53.772 INFO:tasks.rados.rados.0.smithi110.stdout:1035: read oid 13 snap 66 2024-09-14T03:40:53.772 INFO:tasks.rados.rados.0.smithi110.stdout:1035: expect (ObjNum 140 snap 39 seq_num 140) 2024-09-14T03:40:53.772 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 105 2024-09-14T03:40:53.772 INFO:tasks.rados.rados.0.smithi110.stdout:1032: finishing write tid 1 to smithi11075562-23 2024-09-14T03:40:53.772 INFO:tasks.rados.rados.0.smithi110.stdout:1036: seq_num 328 ranges {3162112=696320} 2024-09-14T03:40:53.794 INFO:tasks.rados.rados.0.smithi110.stdout:1036: writing smithi11075562-4 from 3162112 to 3858432 tid 1 2024-09-14T03:40:53.794 INFO:tasks.rados.rados.0.smithi110.stdout:1037: copy_from oid 34 from oid 39 current snap is 105 2024-09-14T03:40:53.794 INFO:tasks.rados.rados.0.smithi110.stdout:1032: finishing write tid 2 to smithi11075562-23 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1032: finishing write tid 3 to smithi11075562-23 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 272 (ObjNum 326 snap 105 seq_num 326) dirty exists 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1032: left oid 23 (ObjNum 326 snap 105 seq_num 326) 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1033: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1033: finishing copy_from to smithi11075562-32 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 555 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1032: done (5 left) 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1033: done (4 left) 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1038: read oid 3 snap -1 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1038: expect (ObjNum 316 snap 102 seq_num 316) 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 105 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1034: finishing write tid 1 to smithi11075562-10 2024-09-14T03:40:53.795 INFO:tasks.rados.rados.0.smithi110.stdout:1039: seq_num 329 ranges {3973120=712704} 2024-09-14T03:40:53.821 INFO:tasks.rados.rados.0.smithi110.stdout:1039: writing smithi11075562-14 from 3973120 to 4685824 tid 1 2024-09-14T03:40:53.821 INFO:tasks.rados.rados.0.smithi110.stdout:1040: read oid 19 snap -1 2024-09-14T03:40:53.821 INFO:tasks.rados.rados.0.smithi110.stdout:1040: expect (ObjNum 323 snap 104 seq_num 323) 2024-09-14T03:40:53.821 INFO:tasks.rados.rados.0.smithi110.stdout:1041: setattr oid 9 current snap is 105 2024-09-14T03:40:53.821 INFO:tasks.rados.rados.0.smithi110.stdout:1034: finishing write tid 2 to smithi11075562-10 2024-09-14T03:40:53.821 INFO:tasks.rados.rados.0.smithi110.stdout:1034: finishing write tid 3 to smithi11075562-10 2024-09-14T03:40:53.821 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 274 (ObjNum 327 snap 105 seq_num 327) dirty exists 2024-09-14T03:40:53.821 INFO:tasks.rados.rados.0.smithi110.stdout:1034: left oid 10 (ObjNum 327 snap 105 seq_num 327) 2024-09-14T03:40:53.822 INFO:tasks.rados.rados.0.smithi110.stdout:1035: expect (ObjNum 140 snap 39 seq_num 140) 2024-09-14T03:40:53.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:53 smithi110 ceph-mon[35374]: Upgrade: Updating mon.c 2024-09-14T03:40:53.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:53 smithi110 ceph-mon[35374]: Deploying daemon mon.c on smithi110 2024-09-14T03:40:53.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:53 smithi110 ceph-mon[35374]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:40:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:53 smithi110 ceph-mon[86423]: Upgrade: Updating mon.c 2024-09-14T03:40:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:53 smithi110 ceph-mon[86423]: Deploying daemon mon.c on smithi110 2024-09-14T03:40:53.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:40:53 smithi110 ceph-mon[86423]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:40:53.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:53 smithi195 ceph-mon[30695]: Upgrade: Updating mon.c 2024-09-14T03:40:53.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:53 smithi195 ceph-mon[30695]: Deploying daemon mon.c on smithi110 2024-09-14T03:40:53.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:40:53 smithi195 ceph-mon[30695]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:40:53.962 INFO:tasks.rados.rados.0.smithi110.stdout:1038: expect (ObjNum 316 snap 102 seq_num 316) 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1036: finishing write tid 1 to smithi11075562-4 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1036: finishing write tid 2 to smithi11075562-4 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1036: finishing write tid 3 to smithi11075562-4 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 314 (ObjNum 328 snap 105 seq_num 328) dirty exists 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1036: left oid 4 (ObjNum 328 snap 105 seq_num 328) 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1037: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1037: finishing copy_from to smithi11075562-34 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 472 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1039: finishing write tid 1 to smithi11075562-14 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1039: finishing write tid 2 to smithi11075562-14 2024-09-14T03:40:53.976 INFO:tasks.rados.rados.0.smithi110.stdout:1039: finishing write tid 3 to smithi11075562-14 2024-09-14T03:40:53.977 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 580 (ObjNum 329 snap 105 seq_num 329) dirty exists 2024-09-14T03:40:53.977 INFO:tasks.rados.rados.0.smithi110.stdout:1039: left oid 14 (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:40:53.977 INFO:tasks.rados.rados.0.smithi110.stdout:1040: expect (ObjNum 323 snap 104 seq_num 323) 2024-09-14T03:40:54.060 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 478 (ObjNum 271 snap 81 seq_num 271) dirty exists 2024-09-14T03:40:54.060 INFO:tasks.rados.rados.0.smithi110.stdout:1034: done (7 left) 2024-09-14T03:40:54.060 INFO:tasks.rados.rados.0.smithi110.stdout:1035: done (6 left) 2024-09-14T03:40:54.060 INFO:tasks.rados.rados.0.smithi110.stdout:1036: done (5 left) 2024-09-14T03:40:54.060 INFO:tasks.rados.rados.0.smithi110.stdout:1037: done (4 left) 2024-09-14T03:40:54.060 INFO:tasks.rados.rados.0.smithi110.stdout:1038: done (3 left) 2024-09-14T03:40:54.061 INFO:tasks.rados.rados.0.smithi110.stdout:1039: done (2 left) 2024-09-14T03:40:54.061 INFO:tasks.rados.rados.0.smithi110.stdout:1040: done (1 left) 2024-09-14T03:40:54.061 INFO:tasks.rados.rados.0.smithi110.stdout:1041: done (0 left) 2024-09-14T03:40:54.061 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 105 2024-09-14T03:40:54.061 INFO:tasks.rados.rados.0.smithi110.stdout:1042: seq_num 330 ranges {548864=737280} 2024-09-14T03:40:54.068 INFO:tasks.rados.rados.0.smithi110.stdout:1042: writing smithi11075562-50 from 548864 to 1286144 tid 1 2024-09-14T03:40:54.068 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 105 2024-09-14T03:40:54.068 INFO:tasks.rados.rados.0.smithi110.stdout:1043: seq_num 331 ranges {1245184=753664} 2024-09-14T03:40:54.081 INFO:tasks.rados.rados.0.smithi110.stdout:1043: writing smithi11075562-39 from 1245184 to 1998848 tid 1 2024-09-14T03:40:54.081 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 105 2024-09-14T03:40:54.081 INFO:tasks.rados.rados.0.smithi110.stdout:1042: finishing write tid 1 to smithi11075562-50 2024-09-14T03:40:54.081 INFO:tasks.rados.rados.0.smithi110.stdout:1042: finishing write tid 2 to smithi11075562-50 2024-09-14T03:40:54.081 INFO:tasks.rados.rados.0.smithi110.stdout:1042: finishing write tid 3 to smithi11075562-50 2024-09-14T03:40:54.081 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 97 (ObjNum 330 snap 105 seq_num 330) dirty exists 2024-09-14T03:40:54.081 INFO:tasks.rados.rados.0.smithi110.stdout:1042: left oid 50 (ObjNum 330 snap 105 seq_num 330) 2024-09-14T03:40:54.081 INFO:tasks.rados.rados.0.smithi110.stdout:1044: seq_num 332 ranges {0=770048} 2024-09-14T03:40:54.087 INFO:tasks.rados.rados.0.smithi110.stdout:1044: writing smithi11075562-43 from 0 to 770048 tid 1 2024-09-14T03:40:54.087 INFO:tasks.rados.rados.0.smithi110.stdout:1042: done (2 left) 2024-09-14T03:40:54.087 INFO:tasks.rados.rados.0.smithi110.stdout:1045: snap_remove snap 102 2024-09-14T03:40:54.273 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:53 smithi110 systemd[1]: Stopping Ceph mon.c for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:40:54.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:54.526 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:54] ENGINE Bus STOPPING 2024-09-14T03:40:54.527 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c[35362]: 2024-09-14T03:40:54.270+0000 7f6f332f1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:40:54.527 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c[35362]: 2024-09-14T03:40:54.270+0000 7f6f332f1640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-14T03:40:54.848 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:54 smithi110 podman[88093]: 2024-09-14 03:40:54.525391189 +0000 UTC m=+0.381451153 container died 8e2894672cd8a7acaae1403b8d1c9bb67e4bdd64f1313d6f1c35cdc31e7650c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=quincy-eb16b97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-14T03:40:54.848 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:40:54.848 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:54] ENGINE Bus STOPPED 2024-09-14T03:40:54.848 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:54] ENGINE Bus STARTING 2024-09-14T03:40:54.848 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:54] ENGINE Serving on http://:::9283 2024-09-14T03:40:54.848 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:40:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:40:54] ENGINE Bus STARTED 2024-09-14T03:40:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 podman[88093]: 2024-09-14 03:40:55.04682156 +0000 UTC m=+0.902881523 container cleanup 8e2894672cd8a7acaae1403b8d1c9bb67e4bdd64f1313d6f1c35cdc31e7650c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, RELEASE=quincy-eb16b97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:40:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 bash[88093]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c 2024-09-14T03:40:55.544 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 podman[88105]: 2024-09-14 03:40:55.282052014 +0000 UTC m=+0.750905278 container remove 8e2894672cd8a7acaae1403b8d1c9bb67e4bdd64f1313d6f1c35cdc31e7650c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:40:55.544 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-14T03:40:55.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 systemd[1]: Stopped Ceph mon.c for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:40:55.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.c.service: Consumed 7.953s CPU time. 2024-09-14T03:40:55.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 systemd[1]: Starting Ceph mon.c for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:40:56.195 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 podman[88243]: 2024-09-14 03:40:55.84552121 +0000 UTC m=+0.016003211 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:40:56.195 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:55 smithi110 podman[88243]: 2024-09-14 03:40:55.941946412 +0000 UTC m=+0.112428452 container create a2cf589f8f7e8dd48b8d0f40d7b583eff63a0f1c8598f95dd1e7a001d8bcf7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-a278231, CEPH_POINT_RELEASE=, ceph=True) 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 podman[88243]: 2024-09-14 03:40:56.119653842 +0000 UTC m=+0.290135847 container init a2cf589f8f7e8dd48b8d0f40d7b583eff63a0f1c8598f95dd1e7a001d8bcf7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 podman[88243]: 2024-09-14 03:40:56.122744567 +0000 UTC m=+0.293226564 container start a2cf589f8f7e8dd48b8d0f40d7b583eff63a0f1c8598f95dd1e7a001d8bcf7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-a278231) 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 2 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: pidfile_write: ignore empty --pid-file 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: load: jerasure load: lrc 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Git sha 0 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: DB SUMMARY 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: DB Session ID: UR5VF9Z9PU3VGDF6PEY9 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:40:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:40:56.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1103 Bytes 2024-09-14T03:40:56.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-14T03:40:56.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 10173137 ; 2024-09-14T03:40:56.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:40:56.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:40:56.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:40:56.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:40:56.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:40:56.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:40:56.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.env: 0x55a1ed6de120 2024-09-14T03:40:56.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:40:56.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.info_log: 0x55a1ee1d9700 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.statistics: (nil) 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.use_fsync: 0 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:40:56.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.db_log_dir: 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.wal_dir: 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:40:56.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.write_buffer_manager: 0x55a1ee1dc1e0 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:40:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.unordered_write: 0 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.row_cache: None 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.wal_filter: None 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:40:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.wal_compression: 0 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:40:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:40:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_open_files: -1 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Compression algorithms supported: 2024-09-14T03:40:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: kZSTD supported: 0 2024-09-14T03:40:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:40:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:40:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:40:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:40:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:40:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:40:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:40:56.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:40:56.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:40:56.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-14T03:40:56.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:40:56.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:40:56.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.merge_operator: 2024-09-14T03:40:56.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_filter: None 2024-09-14T03:40:56.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a1ee1d9320) 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_type: 0 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_block_index_type: 0 2024-09-14T03:40:56.208 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_shortening: 1 2024-09-14T03:40:56.209 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:40:56.209 INFO:journalctl@ceph.mon.c.smithi110.stdout: checksum: 4 2024-09-14T03:40:56.209 INFO:journalctl@ceph.mon.c.smithi110.stdout: no_block_cache: 0 2024-09-14T03:40:56.209 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache: 0x55a1ee1cd610 2024-09-14T03:40:56.209 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:40:56.211 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_options: 2024-09-14T03:40:56.211 INFO:journalctl@ceph.mon.c.smithi110.stdout: capacity : 536870912 2024-09-14T03:40:56.211 INFO:journalctl@ceph.mon.c.smithi110.stdout: num_shard_bits : 4 2024-09-14T03:40:56.211 INFO:journalctl@ceph.mon.c.smithi110.stdout: strict_capacity_limit : 0 2024-09-14T03:40:56.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:40:56.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_compressed: (nil) 2024-09-14T03:40:56.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: persistent_cache: (nil) 2024-09-14T03:40:56.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_size: 4096 2024-09-14T03:40:56.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_size_deviation: 10 2024-09-14T03:40:56.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_restart_interval: 16 2024-09-14T03:40:56.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_block_restart_interval: 1 2024-09-14T03:40:56.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: metadata_block_size: 4096 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: partition_filters: 0 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: use_delta_encoding: 1 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: filter_policy: bloomfilter 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: whole_key_filtering: 1 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: verify_compression: 0 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: format_version: 5 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: enable_index_compression: 1 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_align: 0 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: max_auto_readahead_size: 262144 2024-09-14T03:40:56.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: prepopulate_block_cache: 0 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression: NoCompression 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.num_levels: 7 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:40:56.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:40:56.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:40:56.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:40:56.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:40:56.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.ttl: 2592000 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:40:56.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-14T03:40:56.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:40:56.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13464, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:40:56.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-14T03:40:56.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 507c9d8f-abbd-4524-ac6f-48c48f5087d4 2024-09-14T03:40:56.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285256148546, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-14T03:40:56.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-14T03:40:56.222 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:56.222 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:40:56.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285256170501, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 6305430, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13470, "largest_seqno": 16754, "table_properties": {"data_size": 6290344, "index_size": 10263, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3781, "raw_key_size": 34342, "raw_average_key_size": 23, "raw_value_size": 6259214, "raw_average_value_size": 4192, "num_data_blocks": 446, "num_entries": 1493, "num_filter_entries": 1493, "num_deletions": 294, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285256, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "507c9d8f-abbd-4524-ac6f-48c48f5087d4", "db_session_id": "UR5VF9Z9PU3VGDF6PEY9", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:40:56.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285256170647, "job": 1, "event": "recovery_finished"} 2024-09-14T03:40:56.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a1ee1fce00 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: DB pointer 0x55a1ee2fe000 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** DB Stats ** 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:40:56.223 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: L0 1/0 6.01 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 691.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: L6 1/0 7.79 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: Sum 2/0 13.80 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 691.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 691.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:40:56.224 INFO:journalctl@ceph.mon.c.smithi110.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 691.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative compaction: 0.01 GB write, 242.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval compaction: 0.01 GB write, 242.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:40:56.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: Block cache BinnedLRUCache@0x55a1ee1cd610#2 capacity: 512.00 MB usage: 16.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: starting mon.c rank 2 at public addrs [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] at bind addrs [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: mon.c@-1(???) e3 preinit fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: mon.c@-1(???).mds e7 new map 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: mon.c@-1(???).mds e7 print_map 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: e7 2024-09-14T03:40:56.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: legacy client fscid: 1 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: Filesystem 'cephfs' (1) 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: fs_name cephfs 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: epoch 7 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: created 2024-09-14T03:34:04.160374+0000 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: modified 2024-09-14T03:34:15.891707+0000 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: tableserver 0 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: root 0 2024-09-14T03:40:56.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: session_timeout 60 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: session_autoclose 300 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: max_file_size 1099511627776 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: max_xattr_size 65536 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: required_client_features {} 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: last_failure 0 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: last_failure_osd_epoch 0 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: max_mds 1 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: in 0 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: up {0=24311} 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: failed 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: damaged 2024-09-14T03:40:56.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: stopped 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_pools [3] 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: metadata_pool 2 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: inline_data disabled 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: balancer 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: bal_rank_mask -1 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: standby_count_wanted 0 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: [mds.a{0:24311} state up:active seq 6 addr [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Standby daemons: 2024-09-14T03:40:56.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2024-09-14T03:40:56.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: [mds.b{-1:24310} state up:standby seq 1 addr [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:40:56.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: mon.c@-1(???).osd e266 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-14T03:40:56.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: mon.c@-1(???).osd e266 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:40:56.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: mon.c@-1(???).osd e266 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:40:56.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: mon.c@-1(???).osd e266 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:40:56.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 ceph-mon[88262]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-14T03:40:56.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 bash[88243]: a2cf589f8f7e8dd48b8d0f40d7b583eff63a0f1c8598f95dd1e7a001d8bcf7a7 2024-09-14T03:40:56.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:40:56 smithi110 systemd[1]: Started Ceph mon.c for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:40:57.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:40:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:40:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:00.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:01.852 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:41:01] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.51.0" 2024-09-14T03:41:02.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: mon.c calling monitor election 2024-09-14T03:41:02.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: mon.b calling monitor election 2024-09-14T03:41:02.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: mon.a calling monitor election 2024-09-14T03:41:02.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:41:02.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: monmap epoch 3 2024-09-14T03:41:02.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:41:02.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: last_changed 2024-09-14T03:30:13.577071+0000 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: min_mon_release 17 (quincy) 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: election_strategy: 1 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: mgrmap e42: y(active, since 20s), standbys: x 2024-09-14T03:41:02.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[88262]: overall HEALTH_OK 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: mon.c calling monitor election 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: mon.b calling monitor election 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: mon.a calling monitor election 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: monmap epoch 3 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: last_changed 2024-09-14T03:30:13.577071+0000 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: min_mon_release 17 (quincy) 2024-09-14T03:41:02.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: election_strategy: 1 2024-09-14T03:41:02.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:41:02.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:41:02.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:41:02.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:41:02.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:41:02.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: mgrmap e42: y(active, since 20s), standbys: x 2024-09-14T03:41:02.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:02 smithi110 ceph-mon[86423]: overall HEALTH_OK 2024-09-14T03:41:02.424 INFO:tasks.rados.rados.0.smithi110.stdout:1045: done (2 left) 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:1046: setattr oid 37 current snap is 105 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:1043: finishing write tid 1 to smithi11075562-39 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:1043: finishing write tid 2 to smithi11075562-39 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:1043: finishing write tid 3 to smithi11075562-39 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 583 (ObjNum 331 snap 105 seq_num 331) dirty exists 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:1043: left oid 39 (ObjNum 331 snap 105 seq_num 331) 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:1044: finishing write tid 1 to smithi11075562-43 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:1044: finishing write tid 2 to smithi11075562-43 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:1044: finishing write tid 3 to smithi11075562-43 2024-09-14T03:41:02.425 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 585 (ObjNum 332 snap 105 seq_num 332) dirty exists 2024-09-14T03:41:02.426 INFO:tasks.rados.rados.0.smithi110.stdout:1044: left oid 43 (ObjNum 332 snap 105 seq_num 332) 2024-09-14T03:41:02.426 INFO:tasks.rados.rados.0.smithi110.stdout:1043: done (2 left) 2024-09-14T03:41:02.426 INFO:tasks.rados.rados.0.smithi110.stdout:1044: done (1 left) 2024-09-14T03:41:02.426 INFO:tasks.rados.rados.0.smithi110.stdout:1047: rollback oid 24 current snap is 105 2024-09-14T03:41:02.427 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 103 2024-09-14T03:41:02.427 INFO:tasks.rados.rados.0.smithi110.stdout:1048: read oid 32 snap -1 2024-09-14T03:41:02.427 INFO:tasks.rados.rados.0.smithi110.stdout:1048: expect (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:41:02.427 INFO:tasks.rados.rados.0.smithi110.stdout:1049: snap_create 2024-09-14T03:41:02.429 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 652 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:41:02.432 INFO:tasks.rados.rados.0.smithi110.stdout:1047: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:41:02.434 INFO:tasks.rados.rados.0.smithi110.stdout:1047: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:41:02.434 INFO:tasks.rados.rados.0.smithi110.stdout:1047: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:41:02.434 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 277 (ObjNum 313 snap 100 seq_num 313) dirty exists 2024-09-14T03:41:02.439 INFO:tasks.rados.rados.0.smithi110.stdout:1048: expect (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:41:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: mon.c calling monitor election 2024-09-14T03:41:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: mon.b calling monitor election 2024-09-14T03:41:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: mon.a calling monitor election 2024-09-14T03:41:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:41:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: monmap epoch 3 2024-09-14T03:41:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:41:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: last_changed 2024-09-14T03:30:13.577071+0000 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: min_mon_release 17 (quincy) 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: election_strategy: 1 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: mgrmap e42: y(active, since 20s), standbys: x 2024-09-14T03:41:02.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:02 smithi195 ceph-mon[30695]: overall HEALTH_OK 2024-09-14T03:41:03.424 INFO:tasks.rados.rados.0.smithi110.stdout:1046: done (3 left) 2024-09-14T03:41:03.425 INFO:tasks.rados.rados.0.smithi110.stdout:1047: done (2 left) 2024-09-14T03:41:03.425 INFO:tasks.rados.rados.0.smithi110.stdout:1048: done (1 left) 2024-09-14T03:41:03.425 INFO:tasks.rados.rados.0.smithi110.stdout:1049: done (0 left) 2024-09-14T03:41:03.425 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 106 2024-09-14T03:41:03.425 INFO:tasks.rados.rados.0.smithi110.stdout:1050: seq_num 333 ranges {0=786432} 2024-09-14T03:41:03.429 INFO:tasks.rados.rados.0.smithi110.stdout:1050: writing smithi11075562-37 from 0 to 786432 tid 1 2024-09-14T03:41:03.429 INFO:tasks.rados.rados.0.smithi110.stdout:1051: rmattr oid 2 current snap is 106 2024-09-14T03:41:03.429 INFO:tasks.rados.rados.0.smithi110.stdout:1051: done (1 left) 2024-09-14T03:41:03.429 INFO:tasks.rados.rados.0.smithi110.stdout:1052: read oid 2 snap 99 2024-09-14T03:41:03.429 INFO:tasks.rados.rados.0.smithi110.stdout:1052: expect (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:41:03.429 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 106 2024-09-14T03:41:03.429 INFO:tasks.rados.rados.0.smithi110.stdout:1053: seq_num 334 ranges {3678208=802816} 2024-09-14T03:41:03.454 INFO:tasks.rados.rados.0.smithi110.stdout:1053: writing smithi11075562-1 from 3678208 to 4481024 tid 1 2024-09-14T03:41:03.454 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 106 2024-09-14T03:41:03.455 INFO:tasks.rados.rados.0.smithi110.stdout:1054: seq_num 335 ranges {2473984=417792,2891776=401408} 2024-09-14T03:41:03.470 INFO:tasks.rados.rados.0.smithi110.stdout:1054: writing smithi11075562-32 from 2473984 to 2891776 tid 1 2024-09-14T03:41:03.473 INFO:tasks.rados.rados.0.smithi110.stdout:1054: writing smithi11075562-32 from 2891776 to 3293184 tid 2 2024-09-14T03:41:03.473 INFO:tasks.rados.rados.0.smithi110.stdout:1055: rmattr oid 7 current snap is 106 2024-09-14T03:41:03.473 INFO:tasks.rados.rados.0.smithi110.stdout:1050: finishing write tid 1 to smithi11075562-37 2024-09-14T03:41:03.477 INFO:tasks.rados.rados.0.smithi110.stdout:1050: finishing write tid 2 to smithi11075562-37 2024-09-14T03:41:03.477 INFO:tasks.rados.rados.0.smithi110.stdout:1052: expect (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:1050: finishing write tid 3 to smithi11075562-37 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 655 (ObjNum 333 snap 106 seq_num 333) dirty exists 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:1050: left oid 37 (ObjNum 333 snap 106 seq_num 333) 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:1053: finishing write tid 1 to smithi11075562-1 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:1053: finishing write tid 2 to smithi11075562-1 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:1053: finishing write tid 3 to smithi11075562-1 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 622 (ObjNum 334 snap 106 seq_num 334) dirty exists 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:1053: left oid 1 (ObjNum 334 snap 106 seq_num 334) 2024-09-14T03:41:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:1054: finishing write tid 1 to smithi11075562-32 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1050: done (4 left) 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1052: done (3 left) 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1053: done (2 left) 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1055: done (1 left) 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1056: setattr oid 14 current snap is 106 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1054: finishing write tid 2 to smithi11075562-32 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1054: finishing write tid 3 to smithi11075562-32 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1054: finishing write tid 4 to smithi11075562-32 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 558 (ObjNum 335 snap 106 seq_num 335) dirty exists 2024-09-14T03:41:03.574 INFO:tasks.rados.rados.0.smithi110.stdout:1054: left oid 32 (ObjNum 335 snap 106 seq_num 335) 2024-09-14T03:41:03.575 INFO:tasks.rados.rados.0.smithi110.stdout:1054: done (1 left) 2024-09-14T03:41:03.575 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 106 2024-09-14T03:41:03.575 INFO:tasks.rados.rados.0.smithi110.stdout:1057: seq_num 336 ranges {1531904=434176,1966080=401408} 2024-09-14T03:41:03.585 INFO:tasks.rados.rados.0.smithi110.stdout:1057: writing smithi11075562-16 from 1531904 to 1966080 tid 1 2024-09-14T03:41:03.587 INFO:tasks.rados.rados.0.smithi110.stdout:1057: writing smithi11075562-16 from 1966080 to 2367488 tid 2 2024-09-14T03:41:03.588 INFO:tasks.rados.rados.0.smithi110.stdout:1058: delete oid 4 current snap is 106 2024-09-14T03:41:03.588 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 589 (ObjNum 329 snap 105 seq_num 329) dirty exists 2024-09-14T03:41:03.590 INFO:tasks.rados.rados.0.smithi110.stdout:1057: finishing write tid 1 to smithi11075562-16 2024-09-14T03:41:03.591 INFO:tasks.rados.rados.0.smithi110.stdout:1056: done (2 left) 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1058: done (1 left) 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1059: read oid 5 snap -1 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1059: expect deleted 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1060: rmattr oid 32 current snap is 106 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1060: done (2 left) 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1061: rmattr oid 47 current snap is 106 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1062: setattr oid 21 current snap is 106 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1057: finishing write tid 2 to smithi11075562-16 2024-09-14T03:41:03.592 INFO:tasks.rados.rados.0.smithi110.stdout:1057: finishing write tid 3 to smithi11075562-16 2024-09-14T03:41:03.593 INFO:tasks.rados.rados.0.smithi110.stdout:1059: done (3 left) 2024-09-14T03:41:03.593 INFO:tasks.rados.rados.0.smithi110.stdout:1063: read oid 34 snap 82 2024-09-14T03:41:03.593 INFO:tasks.rados.rados.0.smithi110.stdout:1063: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:41:03.593 INFO:tasks.rados.rados.0.smithi110.stdout:1064: copy_from oid 1 from oid 32 current snap is 106 2024-09-14T03:41:03.593 INFO:tasks.rados.rados.0.smithi110.stdout:1065: copy_from oid 49 from oid 31 current snap is 106 2024-09-14T03:41:03.593 INFO:tasks.rados.rados.0.smithi110.stdout:1066: snap_create 2024-09-14T03:41:03.594 INFO:tasks.rados.rados.0.smithi110.stdout:1057: finishing write tid 4 to smithi11075562-16 2024-09-14T03:41:03.594 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 482 (ObjNum 336 snap 106 seq_num 336) dirty exists 2024-09-14T03:41:03.594 INFO:tasks.rados.rados.0.smithi110.stdout:1057: left oid 16 (ObjNum 336 snap 106 seq_num 336) 2024-09-14T03:41:03.595 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 657 (ObjNum 305 snap 98 seq_num 305) dirty exists 2024-09-14T03:41:03.595 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 591 (ObjNum 304 snap 98 seq_num 304) dirty exists 2024-09-14T03:41:03.597 INFO:tasks.rados.rados.0.smithi110.stdout:1063: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:41:03.618 INFO:tasks.rados.rados.0.smithi110.stdout:1065: finishing copy_from to smithi11075562-49 2024-09-14T03:41:03.618 INFO:tasks.rados.rados.0.smithi110.stdout:1065: got expected ENOENT (src dne) 2024-09-14T03:41:03.618 INFO:tasks.rados.rados.0.smithi110.stdout:1065: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:41:03.618 INFO:tasks.rados.rados.0.smithi110.stdout:1064: finishing copy_from racing read to smithi11075562-1 2024-09-14T03:41:03.618 INFO:tasks.rados.rados.0.smithi110.stdout:1064: finishing copy_from to smithi11075562-1 2024-09-14T03:41:03.618 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 558 (ObjNum 335 snap 106 seq_num 335) dirty exists 2024-09-14T03:41:03.666 INFO:tasks.rados.rados.0.smithi110.stdout:1057: done (6 left) 2024-09-14T03:41:03.666 INFO:tasks.rados.rados.0.smithi110.stdout:1061: done (5 left) 2024-09-14T03:41:03.666 INFO:tasks.rados.rados.0.smithi110.stdout:1062: done (4 left) 2024-09-14T03:41:03.666 INFO:tasks.rados.rados.0.smithi110.stdout:1063: done (3 left) 2024-09-14T03:41:03.666 INFO:tasks.rados.rados.0.smithi110.stdout:1064: done (2 left) 2024-09-14T03:41:03.667 INFO:tasks.rados.rados.0.smithi110.stdout:1065: done (1 left) 2024-09-14T03:41:03.667 INFO:tasks.rados.rados.0.smithi110.stdout:1066: done (0 left) 2024-09-14T03:41:03.667 INFO:tasks.rados.rados.0.smithi110.stdout:1067: read oid 16 snap 103 2024-09-14T03:41:03.667 INFO:tasks.rados.rados.0.smithi110.stdout:1067: expect (ObjNum 234 snap 69 seq_num 234) 2024-09-14T03:41:03.667 INFO:tasks.rados.rados.0.smithi110.stdout:1068: setattr oid 20 current snap is 107 2024-09-14T03:41:03.668 INFO:tasks.rados.rados.0.smithi110.stdout:1069: copy_from oid 23 from oid 36 current snap is 107 2024-09-14T03:41:03.668 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 107 2024-09-14T03:41:03.668 INFO:tasks.rados.rados.0.smithi110.stdout:1070: seq_num 338 ranges {466944=466944,933888=401408} 2024-09-14T03:41:03.672 INFO:tasks.rados.rados.0.smithi110.stdout:1070: writing smithi11075562-17 from 466944 to 933888 tid 1 2024-09-14T03:41:03.675 INFO:tasks.rados.rados.0.smithi110.stdout:1070: writing smithi11075562-17 from 933888 to 1335296 tid 2 2024-09-14T03:41:03.675 INFO:tasks.rados.rados.0.smithi110.stdout:1071: setattr oid 13 current snap is 107 2024-09-14T03:41:03.675 INFO:tasks.rados.rados.0.smithi110.stdout:1067: expect (ObjNum 234 snap 69 seq_num 234) 2024-09-14T03:41:03.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:03 smithi110 ceph-mon[88262]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:41:03.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:03 smithi110 ceph-mon[86423]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:41:03.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:03 smithi195 ceph-mon[30695]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:41:03.718 INFO:tasks.rados.rados.0.smithi110.stdout:1069: finishing copy_from to smithi11075562-23 2024-09-14T03:41:03.718 INFO:tasks.rados.rados.0.smithi110.stdout:1069: got expected ENOENT (src dne) 2024-09-14T03:41:03.718 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 479 (ObjNum 292 snap 90 seq_num 292) dirty exists 2024-09-14T03:41:03.718 INFO:tasks.rados.rados.0.smithi110.stdout:1069: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:41:03.718 INFO:tasks.rados.rados.0.smithi110.stdout:1070: finishing write tid 1 to smithi11075562-17 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1070: finishing write tid 2 to smithi11075562-17 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1070: finishing write tid 3 to smithi11075562-17 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1070: finishing write tid 4 to smithi11075562-17 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 401 (ObjNum 338 snap 107 seq_num 338) dirty exists 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1070: left oid 17 (ObjNum 338 snap 107 seq_num 338) 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1067: done (4 left) 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1068: done (3 left) 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1069: done (2 left) 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1070: done (1 left) 2024-09-14T03:41:03.719 INFO:tasks.rados.rados.0.smithi110.stdout:1072: snap_create 2024-09-14T03:41:03.722 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 625 (ObjNum 287 snap 86 seq_num 287) dirty exists 2024-09-14T03:41:04.422 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:04.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:04 smithi110 ceph-mon[88262]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:41:04.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:04 smithi110 ceph-mon[88262]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:41:04.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:04 smithi110 ceph-mon[88262]: pgmap v29: 121 pgs: 121 active+clean; 247 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-14T03:41:04.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:04 smithi110 ceph-mon[86423]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:41:04.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:04 smithi110 ceph-mon[86423]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:41:04.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:04 smithi110 ceph-mon[86423]: pgmap v29: 121 pgs: 121 active+clean; 247 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-14T03:41:04.677 INFO:tasks.rados.rados.0.smithi110.stdout:1071: done (1 left) 2024-09-14T03:41:04.677 INFO:tasks.rados.rados.0.smithi110.stdout:1072: done (0 left) 2024-09-14T03:41:04.677 INFO:tasks.rados.rados.0.smithi110.stdout:1073: snap_create 2024-09-14T03:41:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:04 smithi195 ceph-mon[30695]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:41:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:04 smithi195 ceph-mon[30695]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:41:04.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:04 smithi195 ceph-mon[30695]: pgmap v29: 121 pgs: 121 active+clean; 247 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-14T03:41:05.675 INFO:tasks.rados.rados.0.smithi110.stdout:1073: done (0 left) 2024-09-14T03:41:05.675 INFO:tasks.rados.rados.0.smithi110.stdout:1074: copy_from oid 10 from oid 16 current snap is 109 2024-09-14T03:41:05.675 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 109 2024-09-14T03:41:05.675 INFO:tasks.rados.rados.0.smithi110.stdout:1075: seq_num 339 ranges {0=483328,483328=401408} 2024-09-14T03:41:05.677 INFO:tasks.rados.rados.0.smithi110.stdout:1075: writing smithi11075562-5 from 0 to 483328 tid 1 2024-09-14T03:41:05.680 INFO:tasks.rados.rados.0.smithi110.stdout:1075: writing smithi11075562-5 from 483328 to 884736 tid 2 2024-09-14T03:41:05.680 INFO:tasks.rados.rados.0.smithi110.stdout:1076: read oid 23 snap 108 2024-09-14T03:41:05.680 INFO:tasks.rados.rados.0.smithi110.stdout:1076: expect (ObjNum 326 snap 105 seq_num 326) 2024-09-14T03:41:05.680 INFO:tasks.rados.rados.0.smithi110.stdout:1077: read oid 47 snap -1 2024-09-14T03:41:05.680 INFO:tasks.rados.rados.0.smithi110.stdout:1077: expect (ObjNum 305 snap 98 seq_num 305) 2024-09-14T03:41:05.681 INFO:tasks.rados.rados.0.smithi110.stdout:1078: snap_create 2024-09-14T03:41:05.687 INFO:tasks.rados.rados.0.smithi110.stdout:1075: finishing write tid 1 to smithi11075562-5 2024-09-14T03:41:05.687 INFO:tasks.rados.rados.0.smithi110.stdout:1075: finishing write tid 2 to smithi11075562-5 2024-09-14T03:41:05.687 INFO:tasks.rados.rados.0.smithi110.stdout:1075: finishing write tid 3 to smithi11075562-5 2024-09-14T03:41:05.692 INFO:tasks.rados.rados.0.smithi110.stdout:1075: finishing write tid 4 to smithi11075562-5 2024-09-14T03:41:05.692 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 660 (ObjNum 339 snap 109 seq_num 339) dirty exists 2024-09-14T03:41:05.692 INFO:tasks.rados.rados.0.smithi110.stdout:1075: left oid 5 (ObjNum 339 snap 109 seq_num 339) 2024-09-14T03:41:05.694 INFO:tasks.rados.rados.0.smithi110.stdout:1077: expect (ObjNum 305 snap 98 seq_num 305) 2024-09-14T03:41:05.768 INFO:tasks.rados.rados.0.smithi110.stdout:1076: expect (ObjNum 326 snap 105 seq_num 326) 2024-09-14T03:41:05.923 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[88262]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:41:05.923 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[88262]: mgrmap e43: y(active, since 22s), standbys: x 2024-09-14T03:41:05.923 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[86423]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[86423]: mgrmap e43: y(active, since 22s), standbys: x 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:05 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:05 smithi195 ceph-mon[30695]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:41:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:05 smithi195 ceph-mon[30695]: mgrmap e43: y(active, since 22s), standbys: x 2024-09-14T03:41:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:05 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:05 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:05 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:05 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:05.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:05.989 INFO:tasks.rados.rados.0.smithi110.stdout:1074: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:41:05.990 INFO:tasks.rados.rados.0.smithi110.stdout:1074: finishing copy_from to smithi11075562-10 2024-09-14T03:41:05.990 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 482 (ObjNum 336 snap 106 seq_num 336) dirty exists 2024-09-14T03:41:06.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:41:06.181+0000 7f940e88b640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-14T03:41:06.681 INFO:tasks.rados.rados.0.smithi110.stdout:1074: done (4 left) 2024-09-14T03:41:06.681 INFO:tasks.rados.rados.0.smithi110.stdout:1075: done (3 left) 2024-09-14T03:41:06.681 INFO:tasks.rados.rados.0.smithi110.stdout:1076: done (2 left) 2024-09-14T03:41:06.681 INFO:tasks.rados.rados.0.smithi110.stdout:1077: done (1 left) 2024-09-14T03:41:06.681 INFO:tasks.rados.rados.0.smithi110.stdout:1078: done (0 left) 2024-09-14T03:41:06.681 INFO:tasks.rados.rados.0.smithi110.stdout:1079: rollback oid 41 current snap is 110 2024-09-14T03:41:06.681 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 105 2024-09-14T03:41:06.682 INFO:tasks.rados.rados.0.smithi110.stdout:1080: delete oid 5 current snap is 110 2024-09-14T03:41:06.687 INFO:tasks.rados.rados.0.smithi110.stdout:1079: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:41:06.687 INFO:tasks.rados.rados.0.smithi110.stdout:1079: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:41:06.687 INFO:tasks.rados.rados.0.smithi110.stdout:1079: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:41:06.688 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 595 (ObjNum 65 snap 0 seq_num 1902474092) dirty exists 2024-09-14T03:41:06.688 INFO:tasks.rados.rados.0.smithi110.stdout:1079: done (1 left) 2024-09-14T03:41:06.688 INFO:tasks.rados.rados.0.smithi110.stdout:1080: done (0 left) 2024-09-14T03:41:06.688 INFO:tasks.rados.rados.0.smithi110.stdout:1081: delete oid 16 current snap is 110 2024-09-14T03:41:06.690 INFO:tasks.rados.rados.0.smithi110.stdout:1081: done (0 left) 2024-09-14T03:41:06.690 INFO:tasks.rados.rados.0.smithi110.stdout:1082: snap_remove snap 95 2024-09-14T03:41:06.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:06 smithi110 ceph-mon[88262]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:41:06.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:06 smithi110 ceph-mon[88262]: pgmap v32: 121 pgs: 121 active+clean; 247 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-14T03:41:06.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:06 smithi110 ceph-mon[86423]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:41:06.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:06 smithi110 ceph-mon[86423]: pgmap v32: 121 pgs: 121 active+clean; 247 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-14T03:41:06.949 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:06 smithi195 ceph-mon[30695]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:41:06.949 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:06 smithi195 ceph-mon[30695]: pgmap v32: 121 pgs: 121 active+clean; 247 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-14T03:41:07.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:07.691 INFO:tasks.rados.rados.0.smithi110.stdout:1082: done (0 left) 2024-09-14T03:41:07.691 INFO:tasks.rados.rados.0.smithi110.stdout:1083: snap_remove snap 100 2024-09-14T03:41:07.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:07 smithi110 ceph-mon[88262]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:41:07.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:07 smithi110 ceph-mon[86423]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:41:07.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:07 smithi195 ceph-mon[30695]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:41:08.667 INFO:tasks.rados.rados.0.smithi110.stdout:1083: done (0 left) 2024-09-14T03:41:08.667 INFO:tasks.rados.rados.0.smithi110.stdout:1084: snap_create 2024-09-14T03:41:08.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:08 smithi110 ceph-mon[86423]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:41:08.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:08 smithi110 ceph-mon[86423]: pgmap v35: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 620 KiB/s wr, 11 op/s 2024-09-14T03:41:08.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:08 smithi110 ceph-mon[86423]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:41:08.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:08 smithi110 ceph-mon[88262]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:41:08.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:08 smithi110 ceph-mon[88262]: pgmap v35: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 620 KiB/s wr, 11 op/s 2024-09-14T03:41:08.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:08 smithi110 ceph-mon[88262]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:41:08.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:08 smithi195 ceph-mon[30695]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:41:08.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:08 smithi195 ceph-mon[30695]: pgmap v35: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 620 KiB/s wr, 11 op/s 2024-09-14T03:41:08.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:08 smithi195 ceph-mon[30695]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:41:09.668 INFO:tasks.rados.rados.0.smithi110.stdout:1084: done (0 left) 2024-09-14T03:41:09.668 INFO:tasks.rados.rados.0.smithi110.stdout:1085: rollback oid 48 current snap is 111 2024-09-14T03:41:09.668 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 48 to 103 2024-09-14T03:41:09.668 INFO:tasks.rados.rados.0.smithi110.stdout:1086: copy_from oid 4 from oid 24 current snap is 111 2024-09-14T03:41:09.669 INFO:tasks.rados.rados.0.smithi110.stdout:1087: snap_create 2024-09-14T03:41:09.671 INFO:tasks.rados.rados.0.smithi110.stdout:1085: finishing rollback tid 0 to smithi11075562-48 2024-09-14T03:41:09.672 INFO:tasks.rados.rados.0.smithi110.stdout:1085: finishing rollback tid 1 to smithi11075562-48 2024-09-14T03:41:09.672 INFO:tasks.rados.rados.0.smithi110.stdout:1085: finishing rollback tid 2 to smithi11075562-48 2024-09-14T03:41:09.672 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 404 (ObjNum 214 snap 65 seq_num 214) dirty exists 2024-09-14T03:41:09.686 INFO:tasks.rados.rados.0.smithi110.stdout:1086: finishing copy_from racing read to smithi11075562-4 2024-09-14T03:41:09.690 INFO:tasks.rados.rados.0.smithi110.stdout:1086: finishing copy_from to smithi11075562-4 2024-09-14T03:41:09.690 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 277 (ObjNum 313 snap 100 seq_num 313) dirty exists 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T03:41:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:09 smithi195 ceph-mon[30695]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:41:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[88262]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:10.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:10.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:10.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:41:10.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:41:10.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T03:41:10.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:09 smithi110 ceph-mon[86423]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:41:10.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:10.669 INFO:tasks.rados.rados.0.smithi110.stdout:1085: done (2 left) 2024-09-14T03:41:10.669 INFO:tasks.rados.rados.0.smithi110.stdout:1086: done (1 left) 2024-09-14T03:41:10.669 INFO:tasks.rados.rados.0.smithi110.stdout:1087: done (0 left) 2024-09-14T03:41:10.670 INFO:tasks.rados.rados.0.smithi110.stdout:1088: read oid 34 snap 78 2024-09-14T03:41:10.670 INFO:tasks.rados.rados.0.smithi110.stdout:1088: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:41:10.670 INFO:tasks.rados.rados.0.smithi110.stdout:1089: rmattr oid 38 current snap is 112 2024-09-14T03:41:10.670 INFO:tasks.rados.rados.0.smithi110.stdout:1090: read oid 27 snap 87 2024-09-14T03:41:10.670 INFO:tasks.rados.rados.0.smithi110.stdout:1090: expect (ObjNum 63263536 snap 21934 seq_num 65154016) 2024-09-14T03:41:10.670 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 112 2024-09-14T03:41:10.670 INFO:tasks.rados.rados.0.smithi110.stdout:1091: seq_num 342 ranges {2146304=532480,2678784=401408} 2024-09-14T03:41:10.684 INFO:tasks.rados.rados.0.smithi110.stdout:1091: writing smithi11075562-9 from 2146304 to 2678784 tid 1 2024-09-14T03:41:10.686 INFO:tasks.rados.rados.0.smithi110.stdout:1091: writing smithi11075562-9 from 2678784 to 3080192 tid 2 2024-09-14T03:41:10.687 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 112 2024-09-14T03:41:10.687 INFO:tasks.rados.rados.0.smithi110.stdout:1092: seq_num 343 ranges {0=548864,548864=401408} 2024-09-14T03:41:10.689 INFO:tasks.rados.rados.0.smithi110.stdout:1092: writing smithi11075562-44 from 0 to 548864 tid 1 2024-09-14T03:41:10.692 INFO:tasks.rados.rados.0.smithi110.stdout:1092: writing smithi11075562-44 from 548864 to 950272 tid 2 2024-09-14T03:41:10.692 INFO:tasks.rados.rados.0.smithi110.stdout:1093: copy_from oid 31 from oid 10 current snap is 112 2024-09-14T03:41:10.692 INFO:tasks.rados.rados.0.smithi110.stdout:1090: expect (ObjNum 63263536 snap 21934 seq_num 65154016) 2024-09-14T03:41:10.696 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 290 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:41:10.696 INFO:tasks.rados.rados.0.smithi110.stdout:1088: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:41:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1091: finishing write tid 1 to smithi11075562-9 2024-09-14T03:41:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1091: finishing write tid 2 to smithi11075562-9 2024-09-14T03:41:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1091: finishing write tid 3 to smithi11075562-9 2024-09-14T03:41:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1091: finishing write tid 4 to smithi11075562-9 2024-09-14T03:41:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 482 (ObjNum 342 snap 112 seq_num 342) dirty exists 2024-09-14T03:41:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1091: left oid 9 (ObjNum 342 snap 112 seq_num 342) 2024-09-14T03:41:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1092: finishing write tid 1 to smithi11075562-44 2024-09-14T03:41:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1088: done (5 left) 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1089: done (4 left) 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1090: done (3 left) 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1091: done (2 left) 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1094: rollback oid 42 current snap is 112 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 72 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1092: finishing write tid 2 to smithi11075562-44 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1092: finishing write tid 3 to smithi11075562-44 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1092: finishing write tid 4 to smithi11075562-44 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 379 (ObjNum 343 snap 112 seq_num 343) dirty exists 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1092: left oid 44 (ObjNum 343 snap 112 seq_num 343) 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1093: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:41:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1093: finishing copy_from to smithi11075562-31 2024-09-14T03:41:10.722 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 482 (ObjNum 336 snap 106 seq_num 336) dirty exists 2024-09-14T03:41:10.722 INFO:tasks.rados.rados.0.smithi110.stdout:1092: done (2 left) 2024-09-14T03:41:10.722 INFO:tasks.rados.rados.0.smithi110.stdout:1093: done (1 left) 2024-09-14T03:41:10.722 INFO:tasks.rados.rados.0.smithi110.stdout:1095: copy_from oid 8 from oid 46 current snap is 112 2024-09-14T03:41:10.722 INFO:tasks.rados.rados.0.smithi110.stdout:1096: read oid 27 snap 108 2024-09-14T03:41:10.722 INFO:tasks.rados.rados.0.smithi110.stdout:1096: expect (ObjNum 64929312 snap 21934 seq_num 64882288) 2024-09-14T03:41:10.722 INFO:tasks.rados.rados.0.smithi110.stdout:1097: delete oid 44 current snap is 112 2024-09-14T03:41:10.722 INFO:tasks.rados.rados.0.smithi110.stdout:1096: expect (ObjNum 64929312 snap 21934 seq_num 64882288) 2024-09-14T03:41:10.728 INFO:tasks.rados.rados.0.smithi110.stdout:1094: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:41:10.728 INFO:tasks.rados.rados.0.smithi110.stdout:1094: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:1094: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 639 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:1094: done (3 left) 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:1096: done (2 left) 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:1097: done (1 left) 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:1098: snap_create 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:1095: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:1095: finishing copy_from to smithi11075562-8 2024-09-14T03:41:10.729 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 477 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:41:10.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:10 smithi195 ceph-mon[30695]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:41:10.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:10 smithi195 ceph-mon[30695]: pgmap v38: 121 pgs: 121 active+clean; 256 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 2.5 MiB/s wr, 34 op/s 2024-09-14T03:41:10.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:10 smithi195 ceph-mon[30695]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:10.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:10 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:41:10.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:10 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:41:10.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:10 smithi195 ceph-mon[30695]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:10.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:10 smithi195 ceph-mon[30695]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:41:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[88262]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:41:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[88262]: pgmap v38: 121 pgs: 121 active+clean; 256 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 2.5 MiB/s wr, 34 op/s 2024-09-14T03:41:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[88262]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[86423]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[86423]: pgmap v38: 121 pgs: 121 active+clean; 256 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 2.5 MiB/s wr, 34 op/s 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:41:11.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:11.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:10 smithi110 ceph-mon[86423]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:41:11.714 INFO:tasks.rados.rados.0.smithi110.stdout:1095: done (1 left) 2024-09-14T03:41:11.715 INFO:tasks.rados.rados.0.smithi110.stdout:1098: done (0 left) 2024-09-14T03:41:11.715 INFO:tasks.rados.rados.0.smithi110.stdout:1099: read oid 35 snap -1 2024-09-14T03:41:11.715 INFO:tasks.rados.rados.0.smithi110.stdout:1099: expect deleted 2024-09-14T03:41:11.715 INFO:tasks.rados.rados.0.smithi110.stdout:1100: setattr oid 32 current snap is 113 2024-09-14T03:41:11.715 INFO:tasks.rados.rados.0.smithi110.stdout:1101: read oid 31 snap 96 2024-09-14T03:41:11.715 INFO:tasks.rados.rados.0.smithi110.stdout:1101: expect deleted 2024-09-14T03:41:11.715 INFO:tasks.rados.rados.0.smithi110.stdout:1102: snap_create 2024-09-14T03:41:11.718 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 562 (ObjNum 335 snap 106 seq_num 335) dirty exists 2024-09-14T03:41:11.921 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:11 smithi195 systemd[1]: Stopping Ceph mon.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:41:11.921 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:11 smithi195 ceph-mon[30695]: Upgrade: Updating mon.b 2024-09-14T03:41:11.921 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:11 smithi195 ceph-mon[30695]: Deploying daemon mon.b on smithi195 2024-09-14T03:41:11.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:11 smithi110 ceph-mon[88262]: Upgrade: Updating mon.b 2024-09-14T03:41:11.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:11 smithi110 ceph-mon[88262]: Deploying daemon mon.b on smithi195 2024-09-14T03:41:11.935 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:41:11] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.51.0" 2024-09-14T03:41:11.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:11 smithi110 ceph-mon[86423]: Upgrade: Updating mon.b 2024-09-14T03:41:11.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:11 smithi110 ceph-mon[86423]: Deploying daemon mon.b on smithi195 2024-09-14T03:41:12.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:41:11] ENGINE Bus STOPPING 2024-09-14T03:41:12.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:41:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:41:12.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:41:12] ENGINE Bus STOPPED 2024-09-14T03:41:12.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:41:12] ENGINE Bus STARTING 2024-09-14T03:41:12.197 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:41:12] ENGINE Serving on http://:::9283 2024-09-14T03:41:12.197 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:12 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:41:12] ENGINE Bus STARTED 2024-09-14T03:41:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b[30691]: 2024-09-14T03:41:11.918+0000 7fbd9650b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:41:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b[30691]: 2024-09-14T03:41:11.918+0000 7fbd9650b640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-14T03:41:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:12 smithi195 podman[82224]: 2024-09-14 03:41:12.072229361 +0000 UTC m=+0.244770151 container died b6415a0de8074139de0298ba92f84c77b3e1fa801bca9b70b2396200ea32a6d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:41:12.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:12 smithi195 podman[82224]: 2024-09-14 03:41:12.427790362 +0000 UTC m=+0.600331191 container cleanup b6415a0de8074139de0298ba92f84c77b3e1fa801bca9b70b2396200ea32a6d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=quincy-eb16b97, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:41:12.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:12 smithi195 bash[82224]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b 2024-09-14T03:41:13.099 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:12 smithi195 podman[82236]: 2024-09-14 03:41:12.745810872 +0000 UTC m=+0.669289645 container remove b6415a0de8074139de0298ba92f84c77b3e1fa801bca9b70b2396200ea32a6d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T03:41:13.099 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:12 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-14T03:41:13.099 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:12 smithi195 systemd[1]: Stopped Ceph mon.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:41:13.100 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:12 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.b.service: Consumed 9.757s CPU time. 2024-09-14T03:41:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 systemd[1]: Starting Ceph mon.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:41:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 podman[82368]: 2024-09-14 03:41:13.210039874 +0000 UTC m=+0.015826056 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:41:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 podman[82368]: 2024-09-14 03:41:13.313954564 +0000 UTC m=+0.119740728 container create 4992d17f309cebba5053cbbdbbf6c09d3a9c105d677e5fa9460e384b56dde1db (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:41:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 podman[82368]: 2024-09-14 03:41:13.59188698 +0000 UTC m=+0.397673145 container init 4992d17f309cebba5053cbbdbbf6c09d3a9c105d677e5fa9460e384b56dde1db (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-a278231, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:41:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 podman[82368]: 2024-09-14 03:41:13.595698092 +0000 UTC m=+0.401484249 container start 4992d17f309cebba5053cbbdbbf6c09d3a9c105d677e5fa9460e384b56dde1db (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 2 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: pidfile_write: ignore empty --pid-file 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: load: jerasure load: lrc 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Git sha 0 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: DB SUMMARY 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: DB Session ID: 982TMCRR7BPZXIV4IEG6 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:41:13.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1103 Bytes 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 11301263 ; 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.env: 0x55c2e3203120 2024-09-14T03:41:13.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.info_log: 0x55c2e4cd7700 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.statistics: (nil) 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.use_fsync: 0 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:41:13.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.db_log_dir: 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.wal_dir: 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:41:13.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.write_buffer_manager: 0x55c2e4cda1e0 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.unordered_write: 0 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:41:13.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:41:13.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:41:13.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.row_cache: None 2024-09-14T03:41:13.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.wal_filter: None 2024-09-14T03:41:13.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:41:13.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:41:13.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:41:13.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:41:13.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.wal_compression: 0 2024-09-14T03:41:13.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:41:13.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:41:13.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_open_files: -1 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:41:13.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Compression algorithms supported: 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: kZSTD supported: 0 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:41:13.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.merge_operator: 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_filter: None 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c2e4cd7320) 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:41:13.970 INFO:journalctl@ceph.mon.b.smithi195.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_type: 0 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: data_block_index_type: 0 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_shortening: 1 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: checksum: 4 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: no_block_cache: 0 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache: 0x55c2e4ccb610 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_options: 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: capacity : 536870912 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: num_shard_bits : 4 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: strict_capacity_limit : 0 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.b.smithi195.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_compressed: (nil) 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: persistent_cache: (nil) 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_size: 4096 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_size_deviation: 10 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_restart_interval: 16 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_block_restart_interval: 1 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: metadata_block_size: 4096 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: partition_filters: 0 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: use_delta_encoding: 1 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: filter_policy: bloomfilter 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: whole_key_filtering: 1 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: verify_compression: 0 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.b.smithi195.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout: format_version: 5 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout: enable_index_compression: 1 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_align: 0 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout: max_auto_readahead_size: 262144 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout: prepopulate_block_cache: 0 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression: NoCompression 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:41:13.973 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.num_levels: 7 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:41:13.974 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:41:13.975 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:41:13.976 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:41:13.977 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:41:13.978 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.ttl: 2592000 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:41:13.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13571, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 32e38cd9-13b9-4162-9372-bf7c2d415bfb 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285273622325, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285273647050, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 6990506, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13576, "largest_seqno": 17280, "table_properties": {"data_size": 6973758, "index_size": 11541, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4165, "raw_key_size": 37794, "raw_average_key_size": 23, "raw_value_size": 6939327, "raw_average_value_size": 4228, "num_data_blocks": 500, "num_entries": 1641, "num_filter_entries": 1641, "num_deletions": 298, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285273, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "32e38cd9-13b9-4162-9372-bf7c2d415bfb", "db_session_id": "982TMCRR7BPZXIV4IEG6", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285273647177, "job": 1, "event": "recovery_finished"} 2024-09-14T03:41:13.980 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c2e4cfae00 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: DB pointer 0x55c2e4dfc000 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** DB Stats ** 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:41:13.981 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** Compaction Stats [default] ** 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: L0 1/0 6.67 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 668.6 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: L6 1/0 7.79 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: Sum 2/0 14.45 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 668.6 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 668.6 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** Compaction Stats [default] ** 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:41:13.982 INFO:journalctl@ceph.mon.b.smithi195.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 668.6 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative compaction: 0.01 GB write, 241.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval compaction: 0.01 GB write, 241.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:41:13.983 INFO:journalctl@ceph.mon.b.smithi195.stdout: Block cache BinnedLRUCache@0x55c2e4ccb610#2 capacity: 512.00 MB usage: 16.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: starting mon.b rank 1 at public addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] at bind addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: mon.b@-1(???) e3 preinit fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: mon.b@-1(???).mds e7 new map 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: mon.b@-1(???).mds e7 print_map 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout: e7 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:41:13.984 INFO:journalctl@ceph.mon.b.smithi195.stdout: legacy client fscid: 1 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: Filesystem 'cephfs' (1) 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: fs_name cephfs 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: epoch 7 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: created 2024-09-14T03:34:04.160374+0000 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: modified 2024-09-14T03:34:15.891707+0000 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: tableserver 0 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: root 0 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: session_timeout 60 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: session_autoclose 300 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: max_file_size 1099511627776 2024-09-14T03:41:13.985 INFO:journalctl@ceph.mon.b.smithi195.stdout: max_xattr_size 65536 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: required_client_features {} 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: last_failure 0 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: last_failure_osd_epoch 0 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: max_mds 1 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: in 0 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: up {0=24311} 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: failed 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: damaged 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: stopped 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: data_pools [3] 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: metadata_pool 2 2024-09-14T03:41:13.986 INFO:journalctl@ceph.mon.b.smithi195.stdout: inline_data disabled 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: balancer 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: bal_rank_mask -1 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: standby_count_wanted 0 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: [mds.a{0:24311} state up:active seq 6 addr [v2:172.21.15.110:6834/2015946497,v1:172.21.15.110:6835/2015946497] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: Standby daemons: 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout: [mds.b{-1:24310} state up:standby seq 1 addr [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: mon.b@-1(???).osd e277 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: mon.b@-1(???).osd e277 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: mon.b@-1(???).osd e277 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:41:13.987 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: mon.b@-1(???).osd e277 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:41:13.988 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 ceph-mon[82388]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-14T03:41:13.988 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 bash[82368]: 4992d17f309cebba5053cbbdbbf6c09d3a9c105d677e5fa9460e384b56dde1db 2024-09-14T03:41:13.988 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:13 smithi195 systemd[1]: Started Ceph mon.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:41:14.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:15.102 INFO:tasks.rados.rados.0.smithi110.stdout:1099: done (3 left) 2024-09-14T03:41:15.102 INFO:tasks.rados.rados.0.smithi110.stdout:1100: done (2 left) 2024-09-14T03:41:15.102 INFO:tasks.rados.rados.0.smithi110.stdout:1101: done (1 left) 2024-09-14T03:41:15.102 INFO:tasks.rados.rados.0.smithi110.stdout:1102: done (0 left) 2024-09-14T03:41:15.102 INFO:tasks.rados.rados.0.smithi110.stdout:1103: read oid 50 snap -1 2024-09-14T03:41:15.102 INFO:tasks.rados.rados.0.smithi110.stdout:1103: expect (ObjNum 330 snap 105 seq_num 330) 2024-09-14T03:41:15.102 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 114 2024-09-14T03:41:15.102 INFO:tasks.rados.rados.0.smithi110.stdout:1104: seq_num 345 ranges {0=581632,581632=401408} 2024-09-14T03:41:15.105 INFO:tasks.rados.rados.0.smithi110.stdout:1104: writing smithi11075562-49 from 0 to 581632 tid 1 2024-09-14T03:41:15.108 INFO:tasks.rados.rados.0.smithi110.stdout:1104: writing smithi11075562-49 from 581632 to 983040 tid 2 2024-09-14T03:41:15.108 INFO:tasks.rados.rados.0.smithi110.stdout:1105: rollback oid 32 current snap is 114 2024-09-14T03:41:15.108 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 83 2024-09-14T03:41:15.108 INFO:tasks.rados.rados.0.smithi110.stdout:1106: delete oid 37 current snap is 114 2024-09-14T03:41:15.108 INFO:tasks.rados.rados.0.smithi110.stdout:1103: expect (ObjNum 330 snap 105 seq_num 330) 2024-09-14T03:41:15.153 INFO:tasks.rados.rados.0.smithi110.stdout:1104: finishing write tid 1 to smithi11075562-49 2024-09-14T03:41:15.154 INFO:tasks.rados.rados.0.smithi110.stdout:1104: finishing write tid 2 to smithi11075562-49 2024-09-14T03:41:15.154 INFO:tasks.rados.rados.0.smithi110.stdout:1104: finishing write tid 3 to smithi11075562-49 2024-09-14T03:41:15.154 INFO:tasks.rados.rados.0.smithi110.stdout:1105: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:41:15.154 INFO:tasks.rados.rados.0.smithi110.stdout:1105: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:41:15.154 INFO:tasks.rados.rados.0.smithi110.stdout:1105: finishing rollback tid 2 to smithi11075562-32 2024-09-14T03:41:15.154 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 566 (ObjNum 211 snap 65 seq_num 211) dirty exists 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:1104: finishing write tid 4 to smithi11075562-49 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 642 (ObjNum 345 snap 114 seq_num 345) dirty exists 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:1104: left oid 49 (ObjNum 345 snap 114 seq_num 345) 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:1103: done (3 left) 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:1104: done (2 left) 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:1105: done (1 left) 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:1106: done (0 left) 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 114 2024-09-14T03:41:15.155 INFO:tasks.rados.rados.0.smithi110.stdout:1107: seq_num 347 ranges {0=614400,614400=401408} 2024-09-14T03:41:15.156 INFO:tasks.rados.rados.0.smithi110.stdout:1107: writing smithi11075562-7 from 0 to 614400 tid 1 2024-09-14T03:41:15.158 INFO:tasks.rados.rados.0.smithi110.stdout:1107: writing smithi11075562-7 from 614400 to 1015808 tid 2 2024-09-14T03:41:15.159 INFO:tasks.rados.rados.0.smithi110.stdout:1108: delete oid 9 current snap is 114 2024-09-14T03:41:15.161 INFO:tasks.rados.rados.0.smithi110.stdout:1107: finishing write tid 1 to smithi11075562-7 2024-09-14T03:41:15.162 INFO:tasks.rados.rados.0.smithi110.stdout:1107: finishing write tid 2 to smithi11075562-7 2024-09-14T03:41:15.162 INFO:tasks.rados.rados.0.smithi110.stdout:1108: done (1 left) 2024-09-14T03:41:15.162 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 114 2024-09-14T03:41:15.162 INFO:tasks.rados.rados.0.smithi110.stdout:1109: seq_num 349 ranges {499712=655360,1155072=401408} 2024-09-14T03:41:15.168 INFO:tasks.rados.rados.0.smithi110.stdout:1109: writing smithi11075562-3 from 499712 to 1155072 tid 1 2024-09-14T03:41:15.170 INFO:tasks.rados.rados.0.smithi110.stdout:1109: writing smithi11075562-3 from 1155072 to 1556480 tid 2 2024-09-14T03:41:15.171 INFO:tasks.rados.rados.0.smithi110.stdout:1110: read oid 38 snap -1 2024-09-14T03:41:15.171 INFO:tasks.rados.rados.0.smithi110.stdout:1110: expect (ObjNum 242 snap 69 seq_num 242) 2024-09-14T03:41:15.171 INFO:tasks.rados.rados.0.smithi110.stdout:1111: snap_remove snap 96 2024-09-14T03:41:15.423 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: pgmap v42: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-14T03:41:15.423 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:15.423 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: mon.b calling monitor election 2024-09-14T03:41:15.423 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: mon.c calling monitor election 2024-09-14T03:41:15.423 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: mon.a calling monitor election 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: monmap epoch 4 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: last_changed 2024-09-14T03:41:14.087402+0000 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: min_mon_release 19 (squid) 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: election_strategy: 1 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:41:15.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:41:15.425 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:41:15.425 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-14T03:41:15.425 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: overall HEALTH_OK 2024-09-14T03:41:15.425 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='' 2024-09-14T03:41:15.425 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:15.425 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: pgmap v42: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-14T03:41:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: mon.b calling monitor election 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: mon.c calling monitor election 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: mon.a calling monitor election 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: monmap epoch 4 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: last_changed 2024-09-14T03:41:14.087402+0000 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: min_mon_release 19 (squid) 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: election_strategy: 1 2024-09-14T03:41:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: overall HEALTH_OK 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='' 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: pgmap v42: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-14T03:41:15.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: mon.b calling monitor election 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: mon.c calling monitor election 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: mon.a calling monitor election 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: monmap epoch 4 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: fsid 5517b266-7249-11ef-bceb-c7b262605968 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: last_changed 2024-09-14T03:41:14.087402+0000 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: created 2024-09-14T03:28:36.548032+0000 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: min_mon_release 19 (squid) 2024-09-14T03:41:15.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: election_strategy: 1 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: 0: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.a 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: 2: [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon.c 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: overall HEALTH_OK 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='' 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:15.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:15 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:16.100 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:15.739Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:16.100 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:16.107 INFO:tasks.rados.rados.0.smithi110.stdout:1111: done (3 left) 2024-09-14T03:41:16.107 INFO:tasks.rados.rados.0.smithi110.stdout:1112: read oid 39 snap -1 2024-09-14T03:41:16.107 INFO:tasks.rados.rados.0.smithi110.stdout:1112: expect (ObjNum 331 snap 105 seq_num 331) 2024-09-14T03:41:16.107 INFO:tasks.rados.rados.0.smithi110.stdout:1113: copy_from oid 17 from oid 22 current snap is 114 2024-09-14T03:41:16.107 INFO:tasks.rados.rados.0.smithi110.stdout:1107: finishing write tid 3 to smithi11075562-7 2024-09-14T03:41:16.107 INFO:tasks.rados.rados.0.smithi110.stdout:1107: finishing write tid 4 to smithi11075562-7 2024-09-14T03:41:16.107 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 293 (ObjNum 347 snap 114 seq_num 347) dirty exists 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:1107: left oid 7 (ObjNum 347 snap 114 seq_num 347) 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:1109: finishing write tid 1 to smithi11075562-3 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:1109: finishing write tid 2 to smithi11075562-3 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:1107: done (4 left) 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 114 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:1109: finishing write tid 3 to smithi11075562-3 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:1109: finishing write tid 4 to smithi11075562-3 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 399 (ObjNum 349 snap 114 seq_num 349) dirty exists 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:1109: left oid 3 (ObjNum 349 snap 114 seq_num 349) 2024-09-14T03:41:16.108 INFO:tasks.rados.rados.0.smithi110.stdout:1110: expect (ObjNum 242 snap 69 seq_num 242) 2024-09-14T03:41:16.368 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:16 smithi195 ceph-mon[82388]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:41:16.369 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:16 smithi195 ceph-mon[82388]: pgmap v45: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-14T03:41:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:16 smithi110 ceph-mon[88262]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:41:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:16 smithi110 ceph-mon[88262]: pgmap v45: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-14T03:41:16.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:16 smithi110 ceph-mon[86423]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:41:16.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:16 smithi110 ceph-mon[86423]: pgmap v45: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-14T03:41:16.893 INFO:tasks.rados.rados.0.smithi110.stdout:1112: expect (ObjNum 331 snap 105 seq_num 331) 2024-09-14T03:41:16.974 INFO:tasks.rados.rados.0.smithi110.stdout:1113: finishing copy_from racing read to smithi11075562-17 2024-09-14T03:41:16.975 INFO:tasks.rados.rados.0.smithi110.stdout:1113: finishing copy_from to smithi11075562-17 2024-09-14T03:41:16.975 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 477 (ObjNum 272 snap 81 seq_num 272) dirty exists 2024-09-14T03:41:16.975 INFO:tasks.rados.rados.0.smithi110.stdout:1114: seq_num 350 ranges {2162688=671744,2834432=401408} 2024-09-14T03:41:16.989 INFO:tasks.rados.rados.0.smithi110.stdout:1114: writing smithi11075562-19 from 2162688 to 2834432 tid 1 2024-09-14T03:41:16.991 INFO:tasks.rados.rados.0.smithi110.stdout:1114: writing smithi11075562-19 from 2834432 to 3235840 tid 2 2024-09-14T03:41:16.991 INFO:tasks.rados.rados.0.smithi110.stdout:1109: done (4 left) 2024-09-14T03:41:16.991 INFO:tasks.rados.rados.0.smithi110.stdout:1110: done (3 left) 2024-09-14T03:41:16.992 INFO:tasks.rados.rados.0.smithi110.stdout:1112: done (2 left) 2024-09-14T03:41:16.992 INFO:tasks.rados.rados.0.smithi110.stdout:1113: done (1 left) 2024-09-14T03:41:16.992 INFO:tasks.rados.rados.0.smithi110.stdout:1115: copy_from oid 22 from oid 8 current snap is 114 2024-09-14T03:41:16.992 INFO:tasks.rados.rados.0.smithi110.stdout:1116: delete oid 30 current snap is 114 2024-09-14T03:41:16.993 INFO:tasks.rados.rados.0.smithi110.stdout:1114: finishing write tid 1 to smithi11075562-19 2024-09-14T03:41:16.994 INFO:tasks.rados.rados.0.smithi110.stdout:1114: finishing write tid 2 to smithi11075562-19 2024-09-14T03:41:16.994 INFO:tasks.rados.rados.0.smithi110.stdout:1114: finishing write tid 3 to smithi11075562-19 2024-09-14T03:41:16.995 INFO:tasks.rados.rados.0.smithi110.stdout:1114: finishing write tid 4 to smithi11075562-19 2024-09-14T03:41:16.995 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 506 (ObjNum 350 snap 114 seq_num 350) dirty exists 2024-09-14T03:41:16.995 INFO:tasks.rados.rados.0.smithi110.stdout:1114: left oid 19 (ObjNum 350 snap 114 seq_num 350) 2024-09-14T03:41:16.995 INFO:tasks.rados.rados.0.smithi110.stdout:1114: done (2 left) 2024-09-14T03:41:16.995 INFO:tasks.rados.rados.0.smithi110.stdout:1116: done (1 left) 2024-09-14T03:41:16.996 INFO:tasks.rados.rados.0.smithi110.stdout:1117: rmattr oid 34 current snap is 114 2024-09-14T03:41:16.996 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 114 2024-09-14T03:41:16.996 INFO:tasks.rados.rados.0.smithi110.stdout:1118: seq_num 352 ranges {1286144=704512,1990656=401408} 2024-09-14T03:41:17.006 INFO:tasks.rados.rados.0.smithi110.stdout:1118: writing smithi11075562-50 from 1286144 to 1990656 tid 1 2024-09-14T03:41:17.009 INFO:tasks.rados.rados.0.smithi110.stdout:1118: writing smithi11075562-50 from 1990656 to 2392064 tid 2 2024-09-14T03:41:17.009 INFO:tasks.rados.rados.0.smithi110.stdout:1119: snap_create 2024-09-14T03:41:17.009 INFO:tasks.rados.rados.0.smithi110.stdout:1115: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:41:17.010 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 483 (ObjNum 171 snap 48 seq_num 171) dirty exists 2024-09-14T03:41:17.010 INFO:tasks.rados.rados.0.smithi110.stdout:1115: finishing copy_from to smithi11075562-22 2024-09-14T03:41:17.010 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 477 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:41:17.011 INFO:tasks.rados.rados.0.smithi110.stdout:1118: finishing write tid 1 to smithi11075562-50 2024-09-14T03:41:17.011 INFO:tasks.rados.rados.0.smithi110.stdout:1118: finishing write tid 2 to smithi11075562-50 2024-09-14T03:41:17.011 INFO:tasks.rados.rados.0.smithi110.stdout:1118: finishing write tid 3 to smithi11075562-50 2024-09-14T03:41:17.012 INFO:tasks.rados.rados.0.smithi110.stdout:1118: finishing write tid 4 to smithi11075562-50 2024-09-14T03:41:17.012 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 101 (ObjNum 352 snap 114 seq_num 352) dirty exists 2024-09-14T03:41:17.012 INFO:tasks.rados.rados.0.smithi110.stdout:1118: left oid 50 (ObjNum 352 snap 114 seq_num 352) 2024-09-14T03:41:17.111 INFO:tasks.rados.rados.0.smithi110.stdout:1115: done (3 left) 2024-09-14T03:41:17.111 INFO:tasks.rados.rados.0.smithi110.stdout:1117: done (2 left) 2024-09-14T03:41:17.111 INFO:tasks.rados.rados.0.smithi110.stdout:1118: done (1 left) 2024-09-14T03:41:17.111 INFO:tasks.rados.rados.0.smithi110.stdout:1119: done (0 left) 2024-09-14T03:41:17.112 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 115 2024-09-14T03:41:17.112 INFO:tasks.rados.rados.0.smithi110.stdout:1120: seq_num 353 ranges {1998848=720896,2719744=401408} 2024-09-14T03:41:17.125 INFO:tasks.rados.rados.0.smithi110.stdout:1120: writing smithi11075562-39 from 1998848 to 2719744 tid 1 2024-09-14T03:41:17.128 INFO:tasks.rados.rados.0.smithi110.stdout:1120: writing smithi11075562-39 from 2719744 to 3121152 tid 2 2024-09-14T03:41:17.128 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 115 2024-09-14T03:41:17.128 INFO:tasks.rados.rados.0.smithi110.stdout:1121: seq_num 354 ranges {770048=737280,1507328=401408} 2024-09-14T03:41:17.136 INFO:tasks.rados.rados.0.smithi110.stdout:1121: writing smithi11075562-43 from 770048 to 1507328 tid 1 2024-09-14T03:41:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:1121: writing smithi11075562-43 from 1507328 to 1908736 tid 2 2024-09-14T03:41:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:1122: read oid 2 snap -1 2024-09-14T03:41:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:1122: expect (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:41:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 115 2024-09-14T03:41:17.139 INFO:tasks.rados.rados.0.smithi110.stdout:1123: seq_num 355 ranges {2875392=753664,3629056=401408} 2024-09-14T03:41:17.158 INFO:tasks.rados.rados.0.smithi110.stdout:1123: writing smithi11075562-11 from 2875392 to 3629056 tid 1 2024-09-14T03:41:17.161 INFO:tasks.rados.rados.0.smithi110.stdout:1123: writing smithi11075562-11 from 3629056 to 4030464 tid 2 2024-09-14T03:41:17.161 INFO:tasks.rados.rados.0.smithi110.stdout:1124: rmattr oid 14 current snap is 115 2024-09-14T03:41:17.161 INFO:tasks.rados.rados.0.smithi110.stdout:1120: finishing write tid 1 to smithi11075562-39 2024-09-14T03:41:17.161 INFO:tasks.rados.rados.0.smithi110.stdout:1125: rmattr oid 5 current snap is 115 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:1120: finishing write tid 2 to smithi11075562-39 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:1120: finishing write tid 3 to smithi11075562-39 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:1120: finishing write tid 4 to smithi11075562-39 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 615 (ObjNum 353 snap 115 seq_num 353) dirty exists 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:1120: left oid 39 (ObjNum 353 snap 115 seq_num 353) 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:1121: finishing write tid 1 to smithi11075562-43 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:1121: finishing write tid 2 to smithi11075562-43 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:1120: done (5 left) 2024-09-14T03:41:17.162 INFO:tasks.rados.rados.0.smithi110.stdout:1126: snap_remove snap 53 2024-09-14T03:41:17.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:17 smithi195 ceph-mon[82388]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:41:17.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:17.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:17 smithi110 ceph-mon[86423]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:41:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:17 smithi110 ceph-mon[88262]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:41:17.789 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:41:18.118 INFO:tasks.rados.rados.0.smithi110.stdout:1126: done (5 left) 2024-09-14T03:41:18.118 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 115 2024-09-14T03:41:18.118 INFO:tasks.rados.rados.0.smithi110.stdout:1127: seq_num 356 ranges {7372800=770048,8142848=401408} 2024-09-14T03:41:18.164 INFO:tasks.rados.rados.0.smithi110.stdout:1127: writing smithi11075562-13 from 7372800 to 8142848 tid 1 2024-09-14T03:41:18.167 INFO:tasks.rados.rados.0.smithi110.stdout:1127: writing smithi11075562-13 from 8142848 to 8544256 tid 2 2024-09-14T03:41:18.167 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 115 2024-09-14T03:41:18.167 INFO:tasks.rados.rados.0.smithi110.stdout:1121: finishing write tid 3 to smithi11075562-43 2024-09-14T03:41:18.167 INFO:tasks.rados.rados.0.smithi110.stdout:1121: finishing write tid 4 to smithi11075562-43 2024-09-14T03:41:18.168 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 619 (ObjNum 354 snap 115 seq_num 354) dirty exists 2024-09-14T03:41:18.168 INFO:tasks.rados.rados.0.smithi110.stdout:1121: left oid 43 (ObjNum 354 snap 115 seq_num 354) 2024-09-14T03:41:18.168 INFO:tasks.rados.rados.0.smithi110.stdout:1122: expect (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:41:18.268 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:41:18.268 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 621 (ObjNum 329 snap 105 seq_num 329) dirty exists 2024-09-14T03:41:18.268 INFO:tasks.rados.rados.0.smithi110.stdout:1123: finishing write tid 1 to smithi11075562-11 2024-09-14T03:41:18.268 INFO:tasks.rados.rados.0.smithi110.stdout:1123: finishing write tid 2 to smithi11075562-11 2024-09-14T03:41:18.268 INFO:tasks.rados.rados.0.smithi110.stdout:1128: seq_num 357 ranges {2367488=786432,3153920=401408} 2024-09-14T03:41:18.284 INFO:tasks.rados.rados.0.smithi110.stdout:1128: writing smithi11075562-31 from 2367488 to 3153920 tid 1 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1128: writing smithi11075562-31 from 3153920 to 3555328 tid 2 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1121: done (6 left) 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1122: done (5 left) 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1124: done (4 left) 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1125: done (3 left) 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1129: snap_create 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1123: finishing write tid 3 to smithi11075562-11 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1123: finishing write tid 4 to smithi11075562-11 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 480 (ObjNum 355 snap 115 seq_num 355) dirty exists 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1123: left oid 11 (ObjNum 355 snap 115 seq_num 355) 2024-09-14T03:41:18.288 INFO:tasks.rados.rados.0.smithi110.stdout:1127: finishing write tid 1 to smithi11075562-13 2024-09-14T03:41:18.289 INFO:tasks.rados.rados.0.smithi110.stdout:1127: finishing write tid 2 to smithi11075562-13 2024-09-14T03:41:18.289 INFO:tasks.rados.rados.0.smithi110.stdout:1127: finishing write tid 3 to smithi11075562-13 2024-09-14T03:41:18.289 INFO:tasks.rados.rados.0.smithi110.stdout:1127: finishing write tid 4 to smithi11075562-13 2024-09-14T03:41:18.289 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 662 (ObjNum 356 snap 115 seq_num 356) dirty exists 2024-09-14T03:41:18.289 INFO:tasks.rados.rados.0.smithi110.stdout:1127: left oid 13 (ObjNum 356 snap 115 seq_num 356) 2024-09-14T03:41:18.290 INFO:tasks.rados.rados.0.smithi110.stdout:1128: finishing write tid 1 to smithi11075562-31 2024-09-14T03:41:18.290 INFO:tasks.rados.rados.0.smithi110.stdout:1128: finishing write tid 2 to smithi11075562-31 2024-09-14T03:41:18.290 INFO:tasks.rados.rados.0.smithi110.stdout:1128: finishing write tid 3 to smithi11075562-31 2024-09-14T03:41:18.291 INFO:tasks.rados.rados.0.smithi110.stdout:1128: finishing write tid 4 to smithi11075562-31 2024-09-14T03:41:18.291 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 486 (ObjNum 357 snap 115 seq_num 357) dirty exists 2024-09-14T03:41:18.291 INFO:tasks.rados.rados.0.smithi110.stdout:1128: left oid 31 (ObjNum 357 snap 115 seq_num 357) 2024-09-14T03:41:18.377 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (119s) 13s ago 6m 16.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (95s) 0s ago 6m 78.7M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (7m) 13s ago 7m 18.9M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (7m) 0s ago 7m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (58s) 0s ago 10m 452M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (85s) 13s ago 12m 516M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (45s) 13s ago 12m 39.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (5s) 0s ago 11m 21.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (22s) 13s ago 11m 27.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:41:18.378 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (114s) 13s ago 5m 9697k - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:41:18.380 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (90s) 0s ago 5m 9722k - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:41:18.380 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (10m) 13s ago 10m 169M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:41:18.380 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (10m) 13s ago 10m 163M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:41:18.380 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (9m) 13s ago 9m 127M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:41:18.380 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (9m) 13s ago 9m 119M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:41:18.380 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (9m) 0s ago 9m 229M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:41:18.380 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (8m) 0s ago 8m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:41:18.380 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (8m) 0s ago 8m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:41:18.381 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (7m) 0s ago 7m 106M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:41:18.381 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (63s) 0s ago 6m 44.2M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:41:18.381 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:18 smithi110 ceph-mon[88262]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:41:18.381 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:18 smithi110 ceph-mon[88262]: pgmap v48: 121 pgs: 121 active+clean; 276 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-14T03:41:18.381 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:18 smithi110 ceph-mon[86423]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:41:18.381 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:18 smithi110 ceph-mon[86423]: pgmap v48: 121 pgs: 121 active+clean; 276 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-14T03:41:18.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:18 smithi195 ceph-mon[82388]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:41:18.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:18 smithi195 ceph-mon[82388]: pgmap v48: 121 pgs: 121 active+clean; 276 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-14T03:41:18.667 INFO:tasks.rados.rados.0.smithi110.stdout:1123: done (3 left) 2024-09-14T03:41:18.668 INFO:tasks.rados.rados.0.smithi110.stdout:1127: done (2 left) 2024-09-14T03:41:18.668 INFO:tasks.rados.rados.0.smithi110.stdout:1128: done (1 left) 2024-09-14T03:41:18.668 INFO:tasks.rados.rados.0.smithi110.stdout:1129: done (0 left) 2024-09-14T03:41:18.668 INFO:tasks.rados.rados.0.smithi110.stdout:1130: read oid 5 snap 92 2024-09-14T03:41:18.668 INFO:tasks.rados.rados.0.smithi110.stdout:1130: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:41:18.668 INFO:tasks.rados.rados.0.smithi110.stdout:1131: read oid 40 snap -1 2024-09-14T03:41:18.668 INFO:tasks.rados.rados.0.smithi110.stdout:1131: expect (ObjNum 258 snap 79 seq_num 258) 2024-09-14T03:41:18.668 INFO:tasks.rados.rados.0.smithi110.stdout:1132: snap_remove snap 87 2024-09-14T03:41:18.716 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:41:18.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:41:18.718 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:41:18.718 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:41:18.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:41:18.718 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:41:18.718 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 10, 2024-09-14T03:41:18.718 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T03:41:18.718 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:41:18.718 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:41:18.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:41:18.657+0000 7f940e88b640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-14T03:41:19.024 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:41:19.024 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:41:19.024 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:41:19.024 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:41:19.024 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:41:19.024 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:41:19.024 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:41:19.025 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:41:19.025 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "5/20 daemons upgraded", 2024-09-14T03:41:19.025 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T03:41:19.025 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:41:19.025 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:41:19.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/459173279' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/459173279' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:19 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.453 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.453 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.453 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:41:19.453 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.453 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/459173279' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.455 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:19 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:19.670 INFO:tasks.rados.rados.0.smithi110.stdout:1132: done (2 left) 2024-09-14T03:41:19.670 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 116 2024-09-14T03:41:19.670 INFO:tasks.rados.rados.0.smithi110.stdout:1133: seq_num 358 ranges {0=802816,802816=401408} 2024-09-14T03:41:19.675 INFO:tasks.rados.rados.0.smithi110.stdout:1133: writing smithi11075562-36 from 0 to 802816 tid 1 2024-09-14T03:41:19.678 INFO:tasks.rados.rados.0.smithi110.stdout:1133: writing smithi11075562-36 from 802816 to 1204224 tid 2 2024-09-14T03:41:19.678 INFO:tasks.rados.rados.0.smithi110.stdout:1134: copy_from oid 44 from oid 26 current snap is 116 2024-09-14T03:41:19.678 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 116 2024-09-14T03:41:19.678 INFO:tasks.rados.rados.0.smithi110.stdout:1135: seq_num 359 ranges {2367488=417792,2785280=417792,3203072=385024} 2024-09-14T03:41:19.694 INFO:tasks.rados.rados.0.smithi110.stdout:1135: writing smithi11075562-10 from 2367488 to 2785280 tid 1 2024-09-14T03:41:19.696 INFO:tasks.rados.rados.0.smithi110.stdout:1135: writing smithi11075562-10 from 2785280 to 3203072 tid 2 2024-09-14T03:41:19.699 INFO:tasks.rados.rados.0.smithi110.stdout:1135: writing smithi11075562-10 from 3203072 to 3588096 tid 3 2024-09-14T03:41:19.699 INFO:tasks.rados.rados.0.smithi110.stdout:1136: snap_remove snap 98 2024-09-14T03:41:20.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:20.677 INFO:tasks.rados.rados.0.smithi110.stdout:1136: done (5 left) 2024-09-14T03:41:20.677 INFO:tasks.rados.rados.0.smithi110.stdout:1137: delete oid 27 current snap is 116 2024-09-14T03:41:20.678 INFO:tasks.rados.rados.0.smithi110.stdout:1130: expect (ObjNum 171 snap 48 seq_num 171) 2024-09-14T03:41:20.728 INFO:tasks.rados.rados.0.smithi110.stdout:1131: expect (ObjNum 258 snap 79 seq_num 258) 2024-09-14T03:41:20.777 INFO:tasks.rados.rados.0.smithi110.stdout:1133: finishing write tid 1 to smithi11075562-36 2024-09-14T03:41:20.777 INFO:tasks.rados.rados.0.smithi110.stdout:1133: finishing write tid 2 to smithi11075562-36 2024-09-14T03:41:20.777 INFO:tasks.rados.rados.0.smithi110.stdout:1133: finishing write tid 3 to smithi11075562-36 2024-09-14T03:41:20.777 INFO:tasks.rados.rados.0.smithi110.stdout:1133: finishing write tid 4 to smithi11075562-36 2024-09-14T03:41:20.777 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 715 (ObjNum 358 snap 116 seq_num 358) dirty exists 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1133: left oid 36 (ObjNum 358 snap 116 seq_num 358) 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1134: finishing copy_from racing read to smithi11075562-44 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1135: finishing write tid 1 to smithi11075562-10 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1135: finishing write tid 2 to smithi11075562-10 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1134: finishing copy_from to smithi11075562-44 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 138 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1135: finishing write tid 3 to smithi11075562-10 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1130: done (5 left) 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1131: done (4 left) 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1133: done (3 left) 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1134: done (2 left) 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1137: done (1 left) 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 34 current snap is 116 2024-09-14T03:41:20.778 INFO:tasks.rados.rados.0.smithi110.stdout:1135: finishing write tid 4 to smithi11075562-10 2024-09-14T03:41:20.779 INFO:tasks.rados.rados.0.smithi110.stdout:1135: finishing write tid 5 to smithi11075562-10 2024-09-14T03:41:20.779 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 486 (ObjNum 359 snap 116 seq_num 359) dirty exists 2024-09-14T03:41:20.779 INFO:tasks.rados.rados.0.smithi110.stdout:1135: left oid 10 (ObjNum 359 snap 116 seq_num 359) 2024-09-14T03:41:20.779 INFO:tasks.rados.rados.0.smithi110.stdout:1138: seq_num 361 ranges {1245184=450560,1695744=581632,2277376=221184} 2024-09-14T03:41:20.785 INFO:tasks.rados.rados.0.smithi110.stdout:1138: writing smithi11075562-34 from 1245184 to 1695744 tid 1 2024-09-14T03:41:20.788 INFO:tasks.rados.rados.0.smithi110.stdout:1138: writing smithi11075562-34 from 1695744 to 2277376 tid 2 2024-09-14T03:41:20.790 INFO:tasks.rados.rados.0.smithi110.stdout:1138: writing smithi11075562-34 from 2277376 to 2498560 tid 3 2024-09-14T03:41:20.791 INFO:tasks.rados.rados.0.smithi110.stdout:1135: done (1 left) 2024-09-14T03:41:20.791 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 116 2024-09-14T03:41:20.791 INFO:tasks.rados.rados.0.smithi110.stdout:1138: finishing write tid 1 to smithi11075562-34 2024-09-14T03:41:20.791 INFO:tasks.rados.rados.0.smithi110.stdout:1139: seq_num 362 ranges {2146304=466944,2613248=466944,3080192=335872} 2024-09-14T03:41:20.803 INFO:tasks.rados.rados.0.smithi110.stdout:1139: writing smithi11075562-15 from 2146304 to 2613248 tid 1 2024-09-14T03:41:20.806 INFO:tasks.rados.rados.0.smithi110.stdout:1139: writing smithi11075562-15 from 2613248 to 3080192 tid 2 2024-09-14T03:41:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:1139: writing smithi11075562-15 from 3080192 to 3416064 tid 3 2024-09-14T03:41:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:1140: read oid 31 snap -1 2024-09-14T03:41:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:1140: expect (ObjNum 357 snap 115 seq_num 357) 2024-09-14T03:41:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 116 2024-09-14T03:41:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:1138: finishing write tid 2 to smithi11075562-34 2024-09-14T03:41:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:1141: seq_num 363 ranges {4816896=483328,5300224=753664,6053888=49152} 2024-09-14T03:41:20.839 INFO:tasks.rados.rados.0.smithi110.stdout:1141: writing smithi11075562-23 from 4816896 to 5300224 tid 1 2024-09-14T03:41:20.843 INFO:tasks.rados.rados.0.smithi110.stdout:1141: writing smithi11075562-23 from 5300224 to 6053888 tid 2 2024-09-14T03:41:20.843 INFO:tasks.rados.rados.0.smithi110.stdout:1141: writing smithi11075562-23 from 6053888 to 6103040 tid 3 2024-09-14T03:41:20.844 INFO:tasks.rados.rados.0.smithi110.stdout:1142: snap_remove snap 112 2024-09-14T03:41:20.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:20 smithi110 ceph-mon[88262]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:20.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:20 smithi110 ceph-mon[88262]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:41:20.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:20 smithi110 ceph-mon[88262]: pgmap v52: 121 pgs: 121 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 3.3 MiB/s wr, 20 op/s 2024-09-14T03:41:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:20 smithi110 ceph-mon[86423]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:20 smithi110 ceph-mon[86423]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:41:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:20 smithi110 ceph-mon[86423]: pgmap v52: 121 pgs: 121 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 3.3 MiB/s wr, 20 op/s 2024-09-14T03:41:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:20 smithi195 ceph-mon[82388]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:20 smithi195 ceph-mon[82388]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:41:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:20 smithi195 ceph-mon[82388]: pgmap v52: 121 pgs: 121 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 3.3 MiB/s wr, 20 op/s 2024-09-14T03:41:21.676 INFO:tasks.rados.rados.0.smithi110.stdout:1142: done (4 left) 2024-09-14T03:41:21.677 INFO:tasks.rados.rados.0.smithi110.stdout:1143: rmattr oid 47 current snap is 116 2024-09-14T03:41:21.677 INFO:tasks.rados.rados.0.smithi110.stdout:1143: done (4 left) 2024-09-14T03:41:21.677 INFO:tasks.rados.rados.0.smithi110.stdout:1144: snap_remove snap 92 2024-09-14T03:41:21.844 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:21 smithi195 ceph-mon[82388]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:41:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:21 smithi110 ceph-mon[88262]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:41:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:21 smithi110 ceph-mon[86423]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:41:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:41:21] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.51.0" 2024-09-14T03:41:22.677 INFO:tasks.rados.rados.0.smithi110.stdout:1144: done (4 left) 2024-09-14T03:41:22.678 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 116 2024-09-14T03:41:22.678 INFO:tasks.rados.rados.0.smithi110.stdout:1138: finishing write tid 3 to smithi11075562-34 2024-09-14T03:41:22.678 INFO:tasks.rados.rados.0.smithi110.stdout:1138: finishing write tid 4 to smithi11075562-34 2024-09-14T03:41:22.678 INFO:tasks.rados.rados.0.smithi110.stdout:1138: finishing write tid 5 to smithi11075562-34 2024-09-14T03:41:22.678 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 490 (ObjNum 361 snap 116 seq_num 361) dirty exists 2024-09-14T03:41:22.678 INFO:tasks.rados.rados.0.smithi110.stdout:1138: left oid 34 (ObjNum 361 snap 116 seq_num 361) 2024-09-14T03:41:22.678 INFO:tasks.rados.rados.0.smithi110.stdout:1139: finishing write tid 1 to smithi11075562-15 2024-09-14T03:41:22.678 INFO:tasks.rados.rados.0.smithi110.stdout:1145: seq_num 364 ranges {581632=499712,1081344=638976,1720320=163840} 2024-09-14T03:41:22.683 INFO:tasks.rados.rados.0.smithi110.stdout:1145: writing smithi11075562-46 from 581632 to 1081344 tid 1 2024-09-14T03:41:22.688 INFO:tasks.rados.rados.0.smithi110.stdout:1145: writing smithi11075562-46 from 1081344 to 1720320 tid 2 2024-09-14T03:41:22.690 INFO:tasks.rados.rados.0.smithi110.stdout:1145: writing smithi11075562-46 from 1720320 to 1884160 tid 3 2024-09-14T03:41:22.690 INFO:tasks.rados.rados.0.smithi110.stdout:1138: done (4 left) 2024-09-14T03:41:22.690 INFO:tasks.rados.rados.0.smithi110.stdout:1146: read oid 11 snap -1 2024-09-14T03:41:22.690 INFO:tasks.rados.rados.0.smithi110.stdout:1146: expect (ObjNum 355 snap 115 seq_num 355) 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1147: copy_from oid 1 from oid 37 current snap is 116 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1148: read oid 42 snap -1 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1148: expect (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1149: rollback oid 17 current snap is 116 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 107 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1150: snap_create 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1139: finishing write tid 2 to smithi11075562-15 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1139: finishing write tid 3 to smithi11075562-15 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1139: finishing write tid 4 to smithi11075562-15 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1139: finishing write tid 5 to smithi11075562-15 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 613 (ObjNum 362 snap 116 seq_num 362) dirty exists 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1139: left oid 15 (ObjNum 362 snap 116 seq_num 362) 2024-09-14T03:41:22.691 INFO:tasks.rados.rados.0.smithi110.stdout:1140: expect (ObjNum 357 snap 115 seq_num 357) 2024-09-14T03:41:22.847 INFO:tasks.rados.rados.0.smithi110.stdout:1141: finishing write tid 1 to smithi11075562-23 2024-09-14T03:41:22.847 INFO:tasks.rados.rados.0.smithi110.stdout:1141: finishing write tid 2 to smithi11075562-23 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1141: finishing write tid 3 to smithi11075562-23 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1141: finishing write tid 4 to smithi11075562-23 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1141: finishing write tid 5 to smithi11075562-23 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 490 (ObjNum 363 snap 116 seq_num 363) dirty exists 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1141: left oid 23 (ObjNum 363 snap 116 seq_num 363) 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1147: finishing copy_from to smithi11075562-1 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1147: got expected ENOENT (src dne) 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1147: finishing copy_from racing read to smithi11075562-1 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1145: finishing write tid 1 to smithi11075562-46 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1145: finishing write tid 2 to smithi11075562-46 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1145: finishing write tid 3 to smithi11075562-46 2024-09-14T03:41:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:1145: finishing write tid 4 to smithi11075562-46 2024-09-14T03:41:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:1145: finishing write tid 5 to smithi11075562-46 2024-09-14T03:41:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 487 (ObjNum 364 snap 116 seq_num 364) dirty exists 2024-09-14T03:41:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:1145: left oid 46 (ObjNum 364 snap 116 seq_num 364) 2024-09-14T03:41:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:1148: expect (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:41:22.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:41:22.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: pgmap v55: 121 pgs: 121 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:22.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:22.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:41:22.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[86423]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: pgmap v55: 121 pgs: 121 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:22.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:22.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:22.916 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:22 smithi110 ceph-mon[88262]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:41:22.934 INFO:tasks.rados.rados.0.smithi110.stdout:1146: expect (ObjNum 355 snap 115 seq_num 355) 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: pgmap v55: 121 pgs: 121 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:22.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:22 smithi195 ceph-mon[82388]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:41:23.113 INFO:tasks.rados.rados.0.smithi110.stdout:1149: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:41:23.114 INFO:tasks.rados.rados.0.smithi110.stdout:1149: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:41:23.114 INFO:tasks.rados.rados.0.smithi110.stdout:1149: finishing rollback tid 2 to smithi11075562-17 2024-09-14T03:41:23.114 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 483 (ObjNum 338 snap 107 seq_num 338) dirty exists 2024-09-14T03:41:23.693 INFO:tasks.rados.rados.0.smithi110.stdout:1139: done (8 left) 2024-09-14T03:41:23.693 INFO:tasks.rados.rados.0.smithi110.stdout:1140: done (7 left) 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1141: done (6 left) 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1145: done (5 left) 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1146: done (4 left) 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1147: done (3 left) 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1148: done (2 left) 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1149: done (1 left) 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1150: done (0 left) 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1151: rollback oid 29 current snap is 117 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 29 to 90 2024-09-14T03:41:23.694 INFO:tasks.rados.rados.0.smithi110.stdout:1152: snap_remove snap 91 2024-09-14T03:41:23.901 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[88262]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[88262]: Reconfiguring daemon mon.a on smithi110 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[86423]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[86423]: Reconfiguring daemon mon.a on smithi110 2024-09-14T03:41:23.902 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:23.903 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:23.903 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:23.903 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:23.903 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:41:23.903 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:23 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:23 smithi195 ceph-mon[82388]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:41:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:23 smithi195 ceph-mon[82388]: Reconfiguring daemon mon.a on smithi110 2024-09-14T03:41:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:23 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:23 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:23 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:23 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:23 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:41:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:23 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:24.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:24 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:24.693 INFO:tasks.rados.rados.0.smithi110.stdout:1152: done (1 left) 2024-09-14T03:41:24.693 INFO:tasks.rados.rados.0.smithi110.stdout:1153: read oid 4 snap -1 2024-09-14T03:41:24.694 INFO:tasks.rados.rados.0.smithi110.stdout:1153: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:24.694 INFO:tasks.rados.rados.0.smithi110.stdout:1154: snap_create 2024-09-14T03:41:24.694 INFO:tasks.rados.rados.0.smithi110.stdout:1151: finishing rollback tid 0 to smithi11075562-29 2024-09-14T03:41:24.694 INFO:tasks.rados.rados.0.smithi110.stdout:1151: finishing rollback tid 1 to smithi11075562-29 2024-09-14T03:41:24.694 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 310 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-14T03:41:24.723 INFO:tasks.rados.rados.0.smithi110.stdout:1153: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[86423]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:41:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[86423]: Reconfiguring daemon mgr.y on smithi110 2024-09-14T03:41:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[86423]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:41:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[86423]: pgmap v58: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.6 MiB/s wr, 26 op/s 2024-09-14T03:41:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[86423]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:41:24.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[88262]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:41:24.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[88262]: Reconfiguring daemon mgr.y on smithi110 2024-09-14T03:41:24.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[88262]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:41:24.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[88262]: pgmap v58: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.6 MiB/s wr, 26 op/s 2024-09-14T03:41:24.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:24.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:24 smithi110 ceph-mon[88262]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:41:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:24 smithi195 ceph-mon[82388]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:41:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:24 smithi195 ceph-mon[82388]: Reconfiguring daemon mgr.y on smithi110 2024-09-14T03:41:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:24 smithi195 ceph-mon[82388]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:41:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:24 smithi195 ceph-mon[82388]: pgmap v58: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.6 MiB/s wr, 26 op/s 2024-09-14T03:41:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:24 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:24 smithi195 ceph-mon[82388]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:41:25.696 INFO:tasks.rados.rados.0.smithi110.stdout:1151: done (2 left) 2024-09-14T03:41:25.696 INFO:tasks.rados.rados.0.smithi110.stdout:1153: done (1 left) 2024-09-14T03:41:25.696 INFO:tasks.rados.rados.0.smithi110.stdout:1154: done (0 left) 2024-09-14T03:41:25.696 INFO:tasks.rados.rados.0.smithi110.stdout:1155: read oid 23 snap -1 2024-09-14T03:41:25.696 INFO:tasks.rados.rados.0.smithi110.stdout:1155: expect (ObjNum 363 snap 116 seq_num 363) 2024-09-14T03:41:25.697 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 118 2024-09-14T03:41:25.697 INFO:tasks.rados.rados.0.smithi110.stdout:1156: seq_num 365 ranges {0=516096,516096=573440,1089536=229376} 2024-09-14T03:41:25.699 INFO:tasks.rados.rados.0.smithi110.stdout:1156: writing smithi11075562-5 from 0 to 516096 tid 1 2024-09-14T03:41:25.702 INFO:tasks.rados.rados.0.smithi110.stdout:1156: writing smithi11075562-5 from 516096 to 1089536 tid 2 2024-09-14T03:41:25.705 INFO:tasks.rados.rados.0.smithi110.stdout:1156: writing smithi11075562-5 from 1089536 to 1318912 tid 3 2024-09-14T03:41:25.705 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 118 2024-09-14T03:41:25.706 INFO:tasks.rados.rados.0.smithi110.stdout:1157: seq_num 366 ranges {983040=532480,1515520=458752,1974272=344064} 2024-09-14T03:41:25.713 INFO:tasks.rados.rados.0.smithi110.stdout:1157: writing smithi11075562-49 from 983040 to 1515520 tid 1 2024-09-14T03:41:25.716 INFO:tasks.rados.rados.0.smithi110.stdout:1157: writing smithi11075562-49 from 1515520 to 1974272 tid 2 2024-09-14T03:41:25.718 INFO:tasks.rados.rados.0.smithi110.stdout:1157: writing smithi11075562-49 from 1974272 to 2318336 tid 3 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1158: read oid 25 snap -1 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1158: expect deleted 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1159: delete oid 48 current snap is 118 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1156: finishing write tid 1 to smithi11075562-5 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1156: finishing write tid 2 to smithi11075562-5 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1156: finishing write tid 3 to smithi11075562-5 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1156: finishing write tid 4 to smithi11075562-5 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1156: finishing write tid 5 to smithi11075562-5 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 755 (ObjNum 365 snap 118 seq_num 365) dirty exists 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1156: left oid 5 (ObjNum 365 snap 118 seq_num 365) 2024-09-14T03:41:25.719 INFO:tasks.rados.rados.0.smithi110.stdout:1155: expect (ObjNum 363 snap 116 seq_num 363) 2024-09-14T03:41:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-mon[82388]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:41:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-mon[82388]: Reconfiguring daemon mon.c on smithi110 2024-09-14T03:41:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-mon[82388]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:41:25.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:25.739Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:25.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:25.972 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:25.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[88262]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:41:25.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:25.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:25.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:25.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[88262]: Reconfiguring daemon mon.c on smithi110 2024-09-14T03:41:25.973 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[88262]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:41:25.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:25.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[86423]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:41:25.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:25.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:25.975 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:25.975 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[86423]: Reconfiguring daemon mon.c on smithi110 2024-09-14T03:41:25.975 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:25 smithi110 ceph-mon[86423]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:41:26.047 INFO:tasks.rados.rados.0.smithi110.stdout:1157: finishing write tid 1 to smithi11075562-49 2024-09-14T03:41:26.047 INFO:tasks.rados.rados.0.smithi110.stdout:1155: done (4 left) 2024-09-14T03:41:26.047 INFO:tasks.rados.rados.0.smithi110.stdout:1156: done (3 left) 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:1158: done (2 left) 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:1159: done (1 left) 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:1160: delete oid 16 current snap is 118 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:1157: finishing write tid 2 to smithi11075562-49 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:1157: finishing write tid 3 to smithi11075562-49 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:1157: finishing write tid 4 to smithi11075562-49 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:1157: finishing write tid 5 to smithi11075562-49 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 693 (ObjNum 366 snap 118 seq_num 366) dirty exists 2024-09-14T03:41:26.048 INFO:tasks.rados.rados.0.smithi110.stdout:1157: left oid 49 (ObjNum 366 snap 118 seq_num 366) 2024-09-14T03:41:26.049 INFO:tasks.rados.rados.0.smithi110.stdout:1157: done (1 left) 2024-09-14T03:41:26.049 INFO:tasks.rados.rados.0.smithi110.stdout:1160: done (0 left) 2024-09-14T03:41:26.049 INFO:tasks.rados.rados.0.smithi110.stdout:1161: snap_create 2024-09-14T03:41:26.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[86423]: pgmap v61: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.6 MiB/s wr, 26 op/s 2024-09-14T03:41:26.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:26.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:26.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:41:26.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:26.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[88262]: pgmap v61: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.6 MiB/s wr, 26 op/s 2024-09-14T03:41:26.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:26.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:26.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:41:26.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:26 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:26.958 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:26 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:26.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:26 smithi195 ceph-mon[82388]: pgmap v61: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.6 MiB/s wr, 26 op/s 2024-09-14T03:41:26.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:26 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:26.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:26 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:26.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:26 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:41:26.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:26 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:27.028 INFO:tasks.rados.rados.0.smithi110.stdout:1161: done (0 left) 2024-09-14T03:41:27.028 INFO:tasks.rados.rados.0.smithi110.stdout:1162: rmattr oid 12 current snap is 119 2024-09-14T03:41:27.028 INFO:tasks.rados.rados.0.smithi110.stdout:1162: done (0 left) 2024-09-14T03:41:27.028 INFO:tasks.rados.rados.0.smithi110.stdout:1163: rollback oid 42 current snap is 119 2024-09-14T03:41:27.029 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 108 2024-09-14T03:41:27.029 INFO:tasks.rados.rados.0.smithi110.stdout:1164: snap_remove snap 84 2024-09-14T03:41:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:27 smithi195 ceph-mon[82388]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:41:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:27 smithi195 ceph-mon[82388]: Reconfiguring daemon osd.0 on smithi110 2024-09-14T03:41:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:27 smithi195 ceph-mon[82388]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:41:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:27 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:27 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:41:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:28.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[88262]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:41:28.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[88262]: Reconfiguring daemon osd.0 on smithi110 2024-09-14T03:41:28.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:28.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[88262]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:41:28.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:28.010 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:28.010 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:41:28.010 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:28.010 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[86423]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:41:28.010 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[86423]: Reconfiguring daemon osd.0 on smithi110 2024-09-14T03:41:28.010 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:28.010 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[86423]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:41:28.011 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:28.011 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:28.011 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:41:28.011 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:28.030 INFO:tasks.rados.rados.0.smithi110.stdout:1164: done (1 left) 2024-09-14T03:41:28.030 INFO:tasks.rados.rados.0.smithi110.stdout:1165: snap_create 2024-09-14T03:41:28.030 INFO:tasks.rados.rados.0.smithi110.stdout:1163: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:41:28.030 INFO:tasks.rados.rados.0.smithi110.stdout:1163: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:41:28.030 INFO:tasks.rados.rados.0.smithi110.stdout:1163: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:41:28.031 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 697 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:41:28.669 INFO:tasks.rados.rados.0.smithi110.stdout:1163: done (1 left) 2024-09-14T03:41:28.669 INFO:tasks.rados.rados.0.smithi110.stdout:1165: done (0 left) 2024-09-14T03:41:28.669 INFO:tasks.rados.rados.0.smithi110.stdout:1166: copy_from oid 23 from oid 38 current snap is 120 2024-09-14T03:41:28.669 INFO:tasks.rados.rados.0.smithi110.stdout:1167: snap_remove snap 82 2024-09-14T03:41:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[88262]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:41:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[88262]: Reconfiguring daemon osd.1 on smithi110 2024-09-14T03:41:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[88262]: pgmap v63: 121 pgs: 121 active+clean; 270 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 320 KiB/s wr, 3 op/s 2024-09-14T03:41:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[88262]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:41:29.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[88262]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:41:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[86423]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:41:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[86423]: Reconfiguring daemon osd.1 on smithi110 2024-09-14T03:41:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[86423]: pgmap v63: 121 pgs: 121 active+clean; 270 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 320 KiB/s wr, 3 op/s 2024-09-14T03:41:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[86423]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:41:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:28 smithi110 ceph-mon[86423]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:41:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:28 smithi195 ceph-mon[82388]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:41:29.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:28 smithi195 ceph-mon[82388]: Reconfiguring daemon osd.1 on smithi110 2024-09-14T03:41:29.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:28 smithi195 ceph-mon[82388]: pgmap v63: 121 pgs: 121 active+clean; 270 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 320 KiB/s wr, 3 op/s 2024-09-14T03:41:29.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:28 smithi195 ceph-mon[82388]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:41:29.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:28 smithi195 ceph-mon[82388]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:41:29.671 INFO:tasks.rados.rados.0.smithi110.stdout:1167: done (1 left) 2024-09-14T03:41:29.671 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 120 2024-09-14T03:41:29.671 INFO:tasks.rados.rados.0.smithi110.stdout:1166: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:41:29.671 INFO:tasks.rados.rados.0.smithi110.stdout:1168: seq_num 369 ranges {0=589824,589824=507904,1097728=294912} 2024-09-14T03:41:29.674 INFO:tasks.rados.rados.0.smithi110.stdout:1168: writing smithi11075562-12 from 0 to 589824 tid 1 2024-09-14T03:41:29.678 INFO:tasks.rados.rados.0.smithi110.stdout:1168: writing smithi11075562-12 from 589824 to 1097728 tid 2 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:1168: writing smithi11075562-12 from 1097728 to 1392640 tid 3 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:1169: copy_from oid 44 from oid 4 current snap is 120 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:1166: finishing copy_from to smithi11075562-23 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 290 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:1166: done (2 left) 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:1170: rmattr oid 15 current snap is 120 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:1170: done (2 left) 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 120 2024-09-14T03:41:29.681 INFO:tasks.rados.rados.0.smithi110.stdout:1171: seq_num 370 ranges {5185536=606208,5791744=794624,6586368=8192} 2024-09-14T03:41:29.713 INFO:tasks.rados.rados.0.smithi110.stdout:1171: writing smithi11075562-26 from 5185536 to 5791744 tid 1 2024-09-14T03:41:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1171: writing smithi11075562-26 from 5791744 to 6586368 tid 2 2024-09-14T03:41:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1171: writing smithi11075562-26 from 6586368 to 6594560 tid 3 2024-09-14T03:41:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1172: snap_create 2024-09-14T03:41:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1168: finishing write tid 1 to smithi11075562-12 2024-09-14T03:41:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1168: finishing write tid 2 to smithi11075562-12 2024-09-14T03:41:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1168: finishing write tid 3 to smithi11075562-12 2024-09-14T03:41:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1168: finishing write tid 4 to smithi11075562-12 2024-09-14T03:41:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1168: finishing write tid 5 to smithi11075562-12 2024-09-14T03:41:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 771 (ObjNum 369 snap 120 seq_num 369) dirty exists 2024-09-14T03:41:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:1168: left oid 12 (ObjNum 369 snap 120 seq_num 369) 2024-09-14T03:41:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:1169: finishing copy_from racing read to smithi11075562-44 2024-09-14T03:41:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:1169: finishing copy_from to smithi11075562-44 2024-09-14T03:41:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 277 (ObjNum 313 snap 100 seq_num 313) dirty exists 2024-09-14T03:41:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:1171: finishing write tid 1 to smithi11075562-26 2024-09-14T03:41:29.721 INFO:tasks.rados.rados.0.smithi110.stdout:1171: finishing write tid 2 to smithi11075562-26 2024-09-14T03:41:29.721 INFO:tasks.rados.rados.0.smithi110.stdout:1171: finishing write tid 3 to smithi11075562-26 2024-09-14T03:41:29.721 INFO:tasks.rados.rados.0.smithi110.stdout:1171: finishing write tid 4 to smithi11075562-26 2024-09-14T03:41:29.721 INFO:tasks.rados.rados.0.smithi110.stdout:1171: finishing write tid 5 to smithi11075562-26 2024-09-14T03:41:29.721 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 142 (ObjNum 370 snap 120 seq_num 370) dirty exists 2024-09-14T03:41:29.721 INFO:tasks.rados.rados.0.smithi110.stdout:1171: left oid 26 (ObjNum 370 snap 120 seq_num 370) 2024-09-14T03:41:30.173 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:30.173 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:30.173 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[88262]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:41:30.173 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:41:30.173 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:30.173 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[88262]: Reconfiguring daemon osd.2 on smithi110 2024-09-14T03:41:30.173 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[88262]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:41:30.173 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[88262]: pgmap v67: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 648 KiB/s wr, 5 op/s 2024-09-14T03:41:30.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:30.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:30.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[86423]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:41:30.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:41:30.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:30.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[86423]: Reconfiguring daemon osd.2 on smithi110 2024-09-14T03:41:30.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[86423]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:41:30.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:29 smithi110 ceph-mon[86423]: pgmap v67: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 648 KiB/s wr, 5 op/s 2024-09-14T03:41:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:29 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:29 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:29 smithi195 ceph-mon[82388]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:41:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:29 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:41:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:29 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:29 smithi195 ceph-mon[82388]: Reconfiguring daemon osd.2 on smithi110 2024-09-14T03:41:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:29 smithi195 ceph-mon[82388]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:41:30.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:29 smithi195 ceph-mon[82388]: pgmap v67: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 648 KiB/s wr, 5 op/s 2024-09-14T03:41:30.676 INFO:tasks.rados.rados.0.smithi110.stdout:1168: done (3 left) 2024-09-14T03:41:30.676 INFO:tasks.rados.rados.0.smithi110.stdout:1169: done (2 left) 2024-09-14T03:41:30.676 INFO:tasks.rados.rados.0.smithi110.stdout:1171: done (1 left) 2024-09-14T03:41:30.676 INFO:tasks.rados.rados.0.smithi110.stdout:1172: done (0 left) 2024-09-14T03:41:30.676 INFO:tasks.rados.rados.0.smithi110.stdout:1173: rollback oid 26 current snap is 121 2024-09-14T03:41:30.676 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 78 2024-09-14T03:41:30.676 INFO:tasks.rados.rados.0.smithi110.stdout:1174: copy_from oid 48 from oid 14 current snap is 121 2024-09-14T03:41:30.676 INFO:tasks.rados.rados.0.smithi110.stdout:1175: snap_create 2024-09-14T03:41:30.682 INFO:tasks.rados.rados.0.smithi110.stdout:1173: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:41:30.682 INFO:tasks.rados.rados.0.smithi110.stdout:1173: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:41:30.682 INFO:tasks.rados.rados.0.smithi110.stdout:1173: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:41:30.682 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 146 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:41:30.702 INFO:tasks.rados.rados.0.smithi110.stdout:1174: finishing copy_from racing read to smithi11075562-48 2024-09-14T03:41:30.705 INFO:tasks.rados.rados.0.smithi110.stdout:1174: finishing copy_from to smithi11075562-48 2024-09-14T03:41:30.705 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 621 (ObjNum 329 snap 105 seq_num 329) dirty exists 2024-09-14T03:41:30.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:31.481 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:31.481 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:31.481 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[88262]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:41:31.481 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:41:31.481 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:31.481 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[88262]: Reconfiguring daemon osd.3 on smithi110 2024-09-14T03:41:31.482 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[88262]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:41:31.482 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:31.482 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:31.482 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[86423]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:41:31.482 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:41:31.482 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:31.482 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[86423]: Reconfiguring daemon osd.3 on smithi110 2024-09-14T03:41:31.482 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-mon[86423]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:41:31.677 INFO:tasks.rados.rados.0.smithi110.stdout:1173: done (2 left) 2024-09-14T03:41:31.677 INFO:tasks.rados.rados.0.smithi110.stdout:1174: done (1 left) 2024-09-14T03:41:31.677 INFO:tasks.rados.rados.0.smithi110.stdout:1175: done (0 left) 2024-09-14T03:41:31.677 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 122 2024-09-14T03:41:31.677 INFO:tasks.rados.rados.0.smithi110.stdout:1176: seq_num 371 ranges {10633216=622592,11255808=679936,11935744=122880} 2024-09-14T03:41:31.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:31 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:31.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:31 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:31.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:31 smithi195 ceph-mon[82388]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:41:31.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:31 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:41:31.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:31 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:31.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:31 smithi195 ceph-mon[82388]: Reconfiguring daemon osd.3 on smithi110 2024-09-14T03:41:31.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:31 smithi195 ceph-mon[82388]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:41:31.738 INFO:tasks.rados.rados.0.smithi110.stdout:1176: writing smithi11075562-23 from 10633216 to 11255808 tid 1 2024-09-14T03:41:31.742 INFO:tasks.rados.rados.0.smithi110.stdout:1176: writing smithi11075562-23 from 11255808 to 11935744 tid 2 2024-09-14T03:41:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:1176: writing smithi11075562-23 from 11935744 to 12058624 tid 3 2024-09-14T03:41:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 122 2024-09-14T03:41:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:1177: seq_num 372 ranges {1204224=638976,1843200=565248,2408448=237568} 2024-09-14T03:41:31.753 INFO:tasks.rados.rados.0.smithi110.stdout:1177: writing smithi11075562-36 from 1204224 to 1843200 tid 1 2024-09-14T03:41:31.756 INFO:tasks.rados.rados.0.smithi110.stdout:1177: writing smithi11075562-36 from 1843200 to 2408448 tid 2 2024-09-14T03:41:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1177: writing smithi11075562-36 from 2408448 to 2646016 tid 3 2024-09-14T03:41:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 122 2024-09-14T03:41:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1176: finishing write tid 1 to smithi11075562-23 2024-09-14T03:41:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:1176: finishing write tid 2 to smithi11075562-23 2024-09-14T03:41:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:1176: finishing write tid 3 to smithi11075562-23 2024-09-14T03:41:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:1176: finishing write tid 4 to smithi11075562-23 2024-09-14T03:41:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:1176: finishing write tid 5 to smithi11075562-23 2024-09-14T03:41:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 494 (ObjNum 371 snap 122 seq_num 371) dirty exists 2024-09-14T03:41:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:1176: left oid 23 (ObjNum 371 snap 122 seq_num 371) 2024-09-14T03:41:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:1177: finishing write tid 1 to smithi11075562-36 2024-09-14T03:41:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:1178: seq_num 373 ranges {1392640=655360,2048000=499712,2547712=303104} 2024-09-14T03:41:31.768 INFO:tasks.rados.rados.0.smithi110.stdout:1178: writing smithi11075562-12 from 1392640 to 2048000 tid 1 2024-09-14T03:41:31.771 INFO:tasks.rados.rados.0.smithi110.stdout:1178: writing smithi11075562-12 from 2048000 to 2547712 tid 2 2024-09-14T03:41:31.773 INFO:tasks.rados.rados.0.smithi110.stdout:1178: writing smithi11075562-12 from 2547712 to 2850816 tid 3 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1177: finishing write tid 2 to smithi11075562-36 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1177: finishing write tid 3 to smithi11075562-36 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1177: finishing write tid 4 to smithi11075562-36 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1177: finishing write tid 5 to smithi11075562-36 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 778 (ObjNum 372 snap 122 seq_num 372) dirty exists 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1177: left oid 36 (ObjNum 372 snap 122 seq_num 372) 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1178: finishing write tid 1 to smithi11075562-12 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1176: done (2 left) 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1177: done (1 left) 2024-09-14T03:41:31.774 INFO:tasks.rados.rados.0.smithi110.stdout:1179: snap_create 2024-09-14T03:41:31.776 INFO:tasks.rados.rados.0.smithi110.stdout:1178: finishing write tid 2 to smithi11075562-12 2024-09-14T03:41:31.776 INFO:tasks.rados.rados.0.smithi110.stdout:1178: finishing write tid 3 to smithi11075562-12 2024-09-14T03:41:31.776 INFO:tasks.rados.rados.0.smithi110.stdout:1178: finishing write tid 4 to smithi11075562-12 2024-09-14T03:41:31.777 INFO:tasks.rados.rados.0.smithi110.stdout:1178: finishing write tid 5 to smithi11075562-12 2024-09-14T03:41:31.777 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 783 (ObjNum 373 snap 122 seq_num 373) dirty exists 2024-09-14T03:41:31.777 INFO:tasks.rados.rados.0.smithi110.stdout:1178: left oid 12 (ObjNum 373 snap 122 seq_num 373) 2024-09-14T03:41:31.907 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:41:31] "GET /metrics HTTP/1.1" 200 36132 "" "Prometheus/2.51.0" 2024-09-14T03:41:32.683 INFO:tasks.rados.rados.0.smithi110.stdout:1178: done (1 left) 2024-09-14T03:41:32.683 INFO:tasks.rados.rados.0.smithi110.stdout:1179: done (0 left) 2024-09-14T03:41:32.683 INFO:tasks.rados.rados.0.smithi110.stdout:1180: read oid 16 snap -1 2024-09-14T03:41:32.683 INFO:tasks.rados.rados.0.smithi110.stdout:1180: expect deleted 2024-09-14T03:41:32.683 INFO:tasks.rados.rados.0.smithi110.stdout:1181: read oid 47 snap -1 2024-09-14T03:41:32.683 INFO:tasks.rados.rados.0.smithi110.stdout:1181: expect (ObjNum 305 snap 98 seq_num 305) 2024-09-14T03:41:32.684 INFO:tasks.rados.rados.0.smithi110.stdout:1182: read oid 24 snap -1 2024-09-14T03:41:32.684 INFO:tasks.rados.rados.0.smithi110.stdout:1182: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:32.684 INFO:tasks.rados.rados.0.smithi110.stdout:1183: read oid 44 snap -1 2024-09-14T03:41:32.684 INFO:tasks.rados.rados.0.smithi110.stdout:1183: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:32.684 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 123 2024-09-14T03:41:32.684 INFO:tasks.rados.rados.0.smithi110.stdout:1184: seq_num 374 ranges {3416064=671744,4087808=778240,4866048=24576} 2024-09-14T03:41:32.705 INFO:tasks.rados.rados.0.smithi110.stdout:1184: writing smithi11075562-15 from 3416064 to 4087808 tid 1 2024-09-14T03:41:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1184: writing smithi11075562-15 from 4087808 to 4866048 tid 2 2024-09-14T03:41:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1184: writing smithi11075562-15 from 4866048 to 4890624 tid 3 2024-09-14T03:41:32.711 INFO:tasks.rados.rados.0.smithi110.stdout:1185: rollback oid 20 current snap is 123 2024-09-14T03:41:32.711 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 111 2024-09-14T03:41:32.711 INFO:tasks.rados.rados.0.smithi110.stdout:1181: expect (ObjNum 305 snap 98 seq_num 305) 2024-09-14T03:41:32.795 INFO:tasks.rados.rados.0.smithi110.stdout:1182: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:32.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: Reconfiguring mds.a (monmap changed)... 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: Reconfiguring daemon mds.a on smithi110 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[88262]: pgmap v70: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:32.873 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:32.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: Reconfiguring mds.a (monmap changed)... 2024-09-14T03:41:32.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:32.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:32.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:32.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: Reconfiguring daemon mds.a on smithi110 2024-09-14T03:41:32.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:41:32.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:32 smithi110 ceph-mon[86423]: pgmap v70: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: Reconfiguring mds.a (monmap changed)... 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: Reconfiguring daemon mds.a on smithi110 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:41:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:32 smithi195 ceph-mon[82388]: pgmap v70: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:32.994 INFO:tasks.rados.rados.0.smithi110.stdout:1184: finishing write tid 1 to smithi11075562-15 2024-09-14T03:41:32.994 INFO:tasks.rados.rados.0.smithi110.stdout:1183: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:33.191 INFO:tasks.rados.rados.0.smithi110.stdout:1185: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:41:33.191 INFO:tasks.rados.rados.0.smithi110.stdout:1184: finishing write tid 2 to smithi11075562-15 2024-09-14T03:41:33.191 INFO:tasks.rados.rados.0.smithi110.stdout:1180: done (5 left) 2024-09-14T03:41:33.191 INFO:tasks.rados.rados.0.smithi110.stdout:1181: done (4 left) 2024-09-14T03:41:33.191 INFO:tasks.rados.rados.0.smithi110.stdout:1182: done (3 left) 2024-09-14T03:41:33.191 INFO:tasks.rados.rados.0.smithi110.stdout:1183: done (2 left) 2024-09-14T03:41:33.191 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 123 2024-09-14T03:41:33.192 INFO:tasks.rados.rados.0.smithi110.stdout:1186: seq_num 375 ranges {3588096=688128,4276224=663552,4939776=139264} 2024-09-14T03:41:33.213 INFO:tasks.rados.rados.0.smithi110.stdout:1186: writing smithi11075562-10 from 3588096 to 4276224 tid 1 2024-09-14T03:41:33.218 INFO:tasks.rados.rados.0.smithi110.stdout:1186: writing smithi11075562-10 from 4276224 to 4939776 tid 2 2024-09-14T03:41:33.218 INFO:tasks.rados.rados.0.smithi110.stdout:1186: writing smithi11075562-10 from 4939776 to 5079040 tid 3 2024-09-14T03:41:33.219 INFO:tasks.rados.rados.0.smithi110.stdout:1187: snap_remove snap 114 2024-09-14T03:41:33.937 INFO:tasks.rados.rados.0.smithi110.stdout:1187: done (3 left) 2024-09-14T03:41:33.937 INFO:tasks.rados.rados.0.smithi110.stdout:1188: snap_remove snap 115 2024-09-14T03:41:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[86423]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:41:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[88262]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:41:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:33 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:33 smithi195 ceph-mon[82388]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:41:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:33 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:33 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:33 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:41:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:33 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:41:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:33 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:34.402 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:34.939 INFO:tasks.rados.rados.0.smithi110.stdout:1188: done (3 left) 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 123 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:1184: finishing write tid 3 to smithi11075562-15 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:1185: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:1184: finishing write tid 4 to smithi11075562-15 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:1185: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 490 (ObjNum 292 snap 90 seq_num 292) dirty exists 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:1184: finishing write tid 5 to smithi11075562-15 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 666 (ObjNum 374 snap 123 seq_num 374) dirty exists 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:1184: left oid 15 (ObjNum 374 snap 123 seq_num 374) 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:1186: finishing write tid 1 to smithi11075562-10 2024-09-14T03:41:34.940 INFO:tasks.rados.rados.0.smithi110.stdout:1186: finishing write tid 2 to smithi11075562-10 2024-09-14T03:41:34.941 INFO:tasks.rados.rados.0.smithi110.stdout:1189: seq_num 376 ranges {0=704512,704512=548864,1253376=253952} 2024-09-14T03:41:34.944 INFO:tasks.rados.rados.0.smithi110.stdout:1189: writing smithi11075562-45 from 0 to 704512 tid 1 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: Reconfiguring daemon mon.b on smithi195 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: pgmap v72: 121 pgs: 121 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.6 MiB/s wr, 12 op/s 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:41:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: Reconfiguring daemon mon.b on smithi195 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: pgmap v72: 121 pgs: 121 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.6 MiB/s wr, 12 op/s 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:41:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:34.948 INFO:tasks.rados.rados.0.smithi110.stdout:1189: writing smithi11075562-45 from 704512 to 1253376 tid 2 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1189: writing smithi11075562-45 from 1253376 to 1507328 tid 3 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1184: done (3 left) 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1185: done (2 left) 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1190: read oid 48 snap -1 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1190: expect (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1191: delete oid 15 current snap is 123 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1186: finishing write tid 3 to smithi11075562-10 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1186: finishing write tid 4 to smithi11075562-10 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:1186: finishing write tid 5 to smithi11075562-10 2024-09-14T03:41:34.950 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 498 (ObjNum 375 snap 123 seq_num 375) dirty exists 2024-09-14T03:41:34.951 INFO:tasks.rados.rados.0.smithi110.stdout:1186: left oid 10 (ObjNum 375 snap 123 seq_num 375) 2024-09-14T03:41:34.951 INFO:tasks.rados.rados.0.smithi110.stdout:1189: finishing write tid 1 to smithi11075562-45 2024-09-14T03:41:34.951 INFO:tasks.rados.rados.0.smithi110.stdout:1189: finishing write tid 2 to smithi11075562-45 2024-09-14T03:41:34.952 INFO:tasks.rados.rados.0.smithi110.stdout:1189: finishing write tid 3 to smithi11075562-45 2024-09-14T03:41:34.953 INFO:tasks.rados.rados.0.smithi110.stdout:1189: finishing write tid 4 to smithi11075562-45 2024-09-14T03:41:34.953 INFO:tasks.rados.rados.0.smithi110.stdout:1189: finishing write tid 5 to smithi11075562-45 2024-09-14T03:41:34.954 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 787 (ObjNum 376 snap 123 seq_num 376) dirty exists 2024-09-14T03:41:34.954 INFO:tasks.rados.rados.0.smithi110.stdout:1189: left oid 45 (ObjNum 376 snap 123 seq_num 376) 2024-09-14T03:41:34.954 INFO:tasks.rados.rados.0.smithi110.stdout:1186: done (3 left) 2024-09-14T03:41:34.954 INFO:tasks.rados.rados.0.smithi110.stdout:1189: done (2 left) 2024-09-14T03:41:34.955 INFO:tasks.rados.rados.0.smithi110.stdout:1191: done (1 left) 2024-09-14T03:41:34.955 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 123 2024-09-14T03:41:34.955 INFO:tasks.rados.rados.0.smithi110.stdout:1192: seq_num 378 ranges {3235840=737280,3973120=720896,4694016=81920} 2024-09-14T03:41:34.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:41:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: Reconfiguring daemon mon.b on smithi195 2024-09-14T03:41:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: pgmap v72: 121 pgs: 121 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.6 MiB/s wr, 12 op/s 2024-09-14T03:41:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:41:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:41:34.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:41:34.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:34.977 INFO:tasks.rados.rados.0.smithi110.stdout:1192: writing smithi11075562-19 from 3235840 to 3973120 tid 1 2024-09-14T03:41:34.980 INFO:tasks.rados.rados.0.smithi110.stdout:1192: writing smithi11075562-19 from 3973120 to 4694016 tid 2 2024-09-14T03:41:34.981 INFO:tasks.rados.rados.0.smithi110.stdout:1192: writing smithi11075562-19 from 4694016 to 4775936 tid 3 2024-09-14T03:41:34.981 INFO:tasks.rados.rados.0.smithi110.stdout:1193: rollback oid 6 current snap is 123 2024-09-14T03:41:34.982 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 6 to 66 2024-09-14T03:41:34.982 INFO:tasks.rados.rados.0.smithi110.stdout:1190: expect (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:41:35.252 INFO:tasks.rados.rados.0.smithi110.stdout:1192: finishing write tid 1 to smithi11075562-19 2024-09-14T03:41:35.252 INFO:tasks.rados.rados.0.smithi110.stdout:1192: finishing write tid 2 to smithi11075562-19 2024-09-14T03:41:35.252 INFO:tasks.rados.rados.0.smithi110.stdout:1192: finishing write tid 3 to smithi11075562-19 2024-09-14T03:41:35.252 INFO:tasks.rados.rados.0.smithi110.stdout:1192: finishing write tid 4 to smithi11075562-19 2024-09-14T03:41:35.252 INFO:tasks.rados.rados.0.smithi110.stdout:1192: finishing write tid 5 to smithi11075562-19 2024-09-14T03:41:35.252 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 560 (ObjNum 378 snap 123 seq_num 378) dirty exists 2024-09-14T03:41:35.252 INFO:tasks.rados.rados.0.smithi110.stdout:1192: left oid 19 (ObjNum 378 snap 123 seq_num 378) 2024-09-14T03:41:35.253 INFO:tasks.rados.rados.0.smithi110.stdout:1193: finishing rollback tid 0 to smithi11075562-6 2024-09-14T03:41:35.253 INFO:tasks.rados.rados.0.smithi110.stdout:1193: finishing rollback tid 1 to smithi11075562-6 2024-09-14T03:41:35.253 INFO:tasks.rados.rados.0.smithi110.stdout:1190: done (2 left) 2024-09-14T03:41:35.253 INFO:tasks.rados.rados.0.smithi110.stdout:1192: done (1 left) 2024-09-14T03:41:35.253 INFO:tasks.rados.rados.0.smithi110.stdout:1194: snap_remove snap 72 2024-09-14T03:41:35.940 INFO:tasks.rados.rados.0.smithi110.stdout:1194: done (1 left) 2024-09-14T03:41:35.940 INFO:tasks.rados.rados.0.smithi110.stdout:1195: read oid 2 snap -1 2024-09-14T03:41:35.940 INFO:tasks.rados.rados.0.smithi110.stdout:1195: expect (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:41:35.940 INFO:tasks.rados.rados.0.smithi110.stdout:1196: snap_remove snap 119 2024-09-14T03:41:35.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:35 smithi110 ceph-mon[86423]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:41:35.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:35 smithi110 ceph-mon[86423]: Reconfiguring daemon mgr.x on smithi195 2024-09-14T03:41:35.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:35 smithi110 ceph-mon[86423]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:41:35.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:35 smithi110 ceph-mon[88262]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:41:35.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:35 smithi110 ceph-mon[88262]: Reconfiguring daemon mgr.x on smithi195 2024-09-14T03:41:35.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:35 smithi110 ceph-mon[88262]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:41:35.958 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:35.958 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:35 smithi195 ceph-mon[82388]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:41:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:35 smithi195 ceph-mon[82388]: Reconfiguring daemon mgr.x on smithi195 2024-09-14T03:41:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:35 smithi195 ceph-mon[82388]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:41:36.894 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:36 smithi195 ceph-mon[82388]: pgmap v75: 121 pgs: 121 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.6 MiB/s wr, 12 op/s 2024-09-14T03:41:36.894 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:36 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:36.894 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:36 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:36.894 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:36 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:41:36.894 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:36 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:36.894 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:36 smithi195 ceph-mon[82388]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:41:36.942 INFO:tasks.rados.rados.0.smithi110.stdout:1196: done (2 left) 2024-09-14T03:41:36.942 INFO:tasks.rados.rados.0.smithi110.stdout:1197: copy_from oid 26 from oid 5 current snap is 123 2024-09-14T03:41:36.943 INFO:tasks.rados.rados.0.smithi110.stdout:1193: finishing rollback tid 2 to smithi11075562-6 2024-09-14T03:41:36.943 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 564 (ObjNum 176 snap 50 seq_num 176) dirty exists 2024-09-14T03:41:36.943 INFO:tasks.rados.rados.0.smithi110.stdout:1195: expect (ObjNum 294 snap 94 seq_num 294) 2024-09-14T03:41:36.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[88262]: pgmap v75: 121 pgs: 121 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.6 MiB/s wr, 12 op/s 2024-09-14T03:41:36.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:36.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:36.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:41:36.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:36.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[88262]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:41:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[86423]: pgmap v75: 121 pgs: 121 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.6 MiB/s wr, 12 op/s 2024-09-14T03:41:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:41:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:36 smithi110 ceph-mon[86423]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:41:37.045 INFO:tasks.rados.rados.0.smithi110.stdout:1197: finishing copy_from racing read to smithi11075562-26 2024-09-14T03:41:37.045 INFO:tasks.rados.rados.0.smithi110.stdout:1197: finishing copy_from to smithi11075562-26 2024-09-14T03:41:37.045 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 755 (ObjNum 365 snap 118 seq_num 365) dirty exists 2024-09-14T03:41:37.045 INFO:tasks.rados.rados.0.smithi110.stdout:1193: done (2 left) 2024-09-14T03:41:37.045 INFO:tasks.rados.rados.0.smithi110.stdout:1195: done (1 left) 2024-09-14T03:41:37.045 INFO:tasks.rados.rados.0.smithi110.stdout:1197: done (0 left) 2024-09-14T03:41:37.045 INFO:tasks.rados.rados.0.smithi110.stdout:1198: snap_create 2024-09-14T03:41:37.188 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:36.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:37.944 INFO:tasks.rados.rados.0.smithi110.stdout:1198: done (0 left) 2024-09-14T03:41:37.944 INFO:tasks.rados.rados.0.smithi110.stdout:1199: delete oid 39 current snap is 124 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[86423]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[86423]: Reconfiguring daemon osd.4 on smithi195 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[86423]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[88262]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:41:37.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[88262]: Reconfiguring daemon osd.4 on smithi195 2024-09-14T03:41:37.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[88262]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:41:37.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:37.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:37.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:41:37.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:37 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:37.948 INFO:tasks.rados.rados.0.smithi110.stdout:1199: done (0 left) 2024-09-14T03:41:37.949 INFO:tasks.rados.rados.0.smithi110.stdout:1200: rollback oid 33 current snap is 124 2024-09-14T03:41:37.949 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 33 to 107 2024-09-14T03:41:37.949 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 124 2024-09-14T03:41:37.949 INFO:tasks.rados.rados.0.smithi110.stdout:1201: seq_num 380 ranges {2850816=770048,3620864=483328,4104192=319488} 2024-09-14T03:41:37.949 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:37 smithi195 ceph-mon[82388]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:41:37.949 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:37 smithi195 ceph-mon[82388]: Reconfiguring daemon osd.4 on smithi195 2024-09-14T03:41:37.949 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:37 smithi195 ceph-mon[82388]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:41:37.949 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:37 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:37.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:37 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:37.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:37 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:41:37.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:37 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:37.968 INFO:tasks.rados.rados.0.smithi110.stdout:1201: writing smithi11075562-12 from 2850816 to 3620864 tid 1 2024-09-14T03:41:37.971 INFO:tasks.rados.rados.0.smithi110.stdout:1201: writing smithi11075562-12 from 3620864 to 4104192 tid 2 2024-09-14T03:41:37.973 INFO:tasks.rados.rados.0.smithi110.stdout:1201: writing smithi11075562-12 from 4104192 to 4423680 tid 3 2024-09-14T03:41:37.973 INFO:tasks.rados.rados.0.smithi110.stdout:1202: rollback oid 42 current snap is 124 2024-09-14T03:41:37.973 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 105 2024-09-14T03:41:37.973 INFO:tasks.rados.rados.0.smithi110.stdout:1200: finishing rollback tid 1 to smithi11075562-33 2024-09-14T03:41:37.973 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 475 (ObjNum 129 snap 0 seq_num 64851536) dirty dne 2024-09-14T03:41:37.973 INFO:tasks.rados.rados.0.smithi110.stdout:1200: done (2 left) 2024-09-14T03:41:37.973 INFO:tasks.rados.rados.0.smithi110.stdout:1203: snap_remove snap 99 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1203: done (2 left) 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1204: read oid 8 snap -1 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1204: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1205: setattr oid 25 current snap is 124 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1201: finishing write tid 1 to smithi11075562-12 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1201: finishing write tid 2 to smithi11075562-12 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1201: finishing write tid 3 to smithi11075562-12 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1201: finishing write tid 4 to smithi11075562-12 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1202: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:41:38.677 INFO:tasks.rados.rados.0.smithi110.stdout:1202: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:41:38.678 INFO:tasks.rados.rados.0.smithi110.stdout:1202: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:41:38.678 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 733 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:41:38.678 INFO:tasks.rados.rados.0.smithi110.stdout:1201: finishing write tid 5 to smithi11075562-12 2024-09-14T03:41:38.678 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 802 (ObjNum 380 snap 124 seq_num 380) dirty exists 2024-09-14T03:41:38.678 INFO:tasks.rados.rados.0.smithi110.stdout:1201: left oid 12 (ObjNum 380 snap 124 seq_num 380) 2024-09-14T03:41:38.678 INFO:tasks.rados.rados.0.smithi110.stdout:1201: done (3 left) 2024-09-14T03:41:38.678 INFO:tasks.rados.rados.0.smithi110.stdout:1202: done (2 left) 2024-09-14T03:41:38.678 INFO:tasks.rados.rados.0.smithi110.stdout:1206: delete oid 3 current snap is 124 2024-09-14T03:41:38.689 INFO:tasks.rados.rados.0.smithi110.stdout:1204: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:41:38.710 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 807 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:41:38.710 INFO:tasks.rados.rados.0.smithi110.stdout:1204: done (2 left) 2024-09-14T03:41:38.711 INFO:tasks.rados.rados.0.smithi110.stdout:1205: done (1 left) 2024-09-14T03:41:38.711 INFO:tasks.rados.rados.0.smithi110.stdout:1206: done (0 left) 2024-09-14T03:41:38.711 INFO:tasks.rados.rados.0.smithi110.stdout:1207: snap_create 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: Reconfiguring daemon osd.5 on smithi195 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: pgmap v78: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:41:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:38 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:41:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: Reconfiguring daemon osd.5 on smithi195 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: pgmap v78: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: Reconfiguring daemon osd.5 on smithi195 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: pgmap v78: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-14T03:41:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:41:39.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:39.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:39.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:41:39.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:41:39.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:38 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:39.708 INFO:tasks.rados.rados.0.smithi110.stdout:1207: done (0 left) 2024-09-14T03:41:39.708 INFO:tasks.rados.rados.0.smithi110.stdout:1208: rollback oid 41 current snap is 125 2024-09-14T03:41:39.708 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 124 2024-09-14T03:41:39.709 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 125 2024-09-14T03:41:39.709 INFO:tasks.rados.rados.0.smithi110.stdout:1209: seq_num 382 ranges {4120576=802816,4923392=704512,5627904=98304} 2024-09-14T03:41:39.735 INFO:tasks.rados.rados.0.smithi110.stdout:1209: writing smithi11075562-4 from 4120576 to 4923392 tid 1 2024-09-14T03:41:39.739 INFO:tasks.rados.rados.0.smithi110.stdout:1209: writing smithi11075562-4 from 4923392 to 5627904 tid 2 2024-09-14T03:41:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:1209: writing smithi11075562-4 from 5627904 to 5726208 tid 3 2024-09-14T03:41:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:1210: delete oid 19 current snap is 125 2024-09-14T03:41:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:1208: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:41:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:1208: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:41:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:1208: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:41:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 699 (ObjNum 129 snap 0 seq_num 65420592) dirty exists 2024-09-14T03:41:39.742 INFO:tasks.rados.rados.0.smithi110.stdout:1208: done (2 left) 2024-09-14T03:41:39.742 INFO:tasks.rados.rados.0.smithi110.stdout:1210: done (1 left) 2024-09-14T03:41:39.742 INFO:tasks.rados.rados.0.smithi110.stdout:1211: rollback oid 6 current snap is 125 2024-09-14T03:41:39.742 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 6 to 107 2024-09-14T03:41:39.742 INFO:tasks.rados.rados.0.smithi110.stdout:1212: snap_remove snap 118 2024-09-14T03:41:39.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:39 smithi195 ceph-mon[82388]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:41:39.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:39 smithi195 ceph-mon[82388]: Reconfiguring daemon osd.6 on smithi195 2024-09-14T03:41:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:39 smithi110 ceph-mon[88262]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:41:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:39 smithi110 ceph-mon[88262]: Reconfiguring daemon osd.6 on smithi195 2024-09-14T03:41:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:39 smithi110 ceph-mon[86423]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:41:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:39 smithi110 ceph-mon[86423]: Reconfiguring daemon osd.6 on smithi195 2024-09-14T03:41:40.402 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:40.714 INFO:tasks.rados.rados.0.smithi110.stdout:1209: finishing write tid 1 to smithi11075562-4 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1209: finishing write tid 2 to smithi11075562-4 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1209: finishing write tid 3 to smithi11075562-4 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1209: finishing write tid 4 to smithi11075562-4 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1209: finishing write tid 5 to smithi11075562-4 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 494 (ObjNum 382 snap 125 seq_num 382) dirty exists 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1209: left oid 4 (ObjNum 382 snap 125 seq_num 382) 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1211: finishing rollback tid 0 to smithi11075562-6 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1211: finishing rollback tid 1 to smithi11075562-6 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1211: finishing rollback tid 2 to smithi11075562-6 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 590 (ObjNum 244 snap 69 seq_num 244) dirty exists 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1209: done (2 left) 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1211: done (1 left) 2024-09-14T03:41:40.715 INFO:tasks.rados.rados.0.smithi110.stdout:1212: done (0 left) 2024-09-14T03:41:40.716 INFO:tasks.rados.rados.0.smithi110.stdout:1213: read oid 36 snap -1 2024-09-14T03:41:40.716 INFO:tasks.rados.rados.0.smithi110.stdout:1213: expect (ObjNum 372 snap 122 seq_num 372) 2024-09-14T03:41:40.716 INFO:tasks.rados.rados.0.smithi110.stdout:1214: delete oid 1 current snap is 125 2024-09-14T03:41:40.721 INFO:tasks.rados.rados.0.smithi110.stdout:1214: done (1 left) 2024-09-14T03:41:40.721 INFO:tasks.rados.rados.0.smithi110.stdout:1215: rollback oid 16 current snap is 125 2024-09-14T03:41:40.721 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 120 2024-09-14T03:41:40.721 INFO:tasks.rados.rados.0.smithi110.stdout:1216: rmattr oid 44 current snap is 125 2024-09-14T03:41:40.721 INFO:tasks.rados.rados.0.smithi110.stdout:1217: delete oid 18 current snap is 125 2024-09-14T03:41:40.724 INFO:tasks.rados.rados.0.smithi110.stdout:1215: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:41:40.725 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 484 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:41:40.725 INFO:tasks.rados.rados.0.smithi110.stdout:1215: done (3 left) 2024-09-14T03:41:40.725 INFO:tasks.rados.rados.0.smithi110.stdout:1217: done (2 left) 2024-09-14T03:41:40.725 INFO:tasks.rados.rados.0.smithi110.stdout:1218: read oid 41 snap -1 2024-09-14T03:41:40.725 INFO:tasks.rados.rados.0.smithi110.stdout:1218: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:41:40.725 INFO:tasks.rados.rados.0.smithi110.stdout:1219: setattr oid 46 current snap is 125 2024-09-14T03:41:40.726 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 381 (ObjNum 313 snap 100 seq_num 313) dirty exists 2024-09-14T03:41:40.726 INFO:tasks.rados.rados.0.smithi110.stdout:1218: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:41:40.730 INFO:tasks.rados.rados.0.smithi110.stdout:1216: done (3 left) 2024-09-14T03:41:40.731 INFO:tasks.rados.rados.0.smithi110.stdout:1218: done (2 left) 2024-09-14T03:41:40.731 INFO:tasks.rados.rados.0.smithi110.stdout:1220: read oid 7 snap -1 2024-09-14T03:41:40.731 INFO:tasks.rados.rados.0.smithi110.stdout:1220: expect (ObjNum 347 snap 114 seq_num 347) 2024-09-14T03:41:40.731 INFO:tasks.rados.rados.0.smithi110.stdout:1221: snap_remove snap 66 2024-09-14T03:41:41.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:40 smithi195 ceph-mon[82388]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:41:41.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:40 smithi195 ceph-mon[82388]: pgmap v82: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-14T03:41:41.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:40 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:40 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:40 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:41:41.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:40 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[86423]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:41:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[86423]: pgmap v82: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-14T03:41:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:41:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:41.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[88262]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:41:41.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[88262]: pgmap v82: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-14T03:41:41.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:41:41.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:40 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:41.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 491 (ObjNum 364 snap 116 seq_num 364) dirty exists 2024-09-14T03:41:41.720 INFO:tasks.rados.rados.0.smithi110.stdout:1220: expect (ObjNum 347 snap 114 seq_num 347) 2024-09-14T03:41:41.755 INFO:tasks.rados.rados.0.smithi110.stdout:1213: expect (ObjNum 372 snap 122 seq_num 372) 2024-09-14T03:41:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:1213: done (3 left) 2024-09-14T03:41:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:1219: done (2 left) 2024-09-14T03:41:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:1220: done (1 left) 2024-09-14T03:41:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:1221: done (0 left) 2024-09-14T03:41:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:1222: read oid 41 snap -1 2024-09-14T03:41:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:1222: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:41:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:1223: rmattr oid 46 current snap is 125 2024-09-14T03:41:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:1224: snap_create 2024-09-14T03:41:41.843 INFO:tasks.rados.rados.0.smithi110.stdout:1222: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:41:41.846 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 492 (ObjNum 364 snap 116 seq_num 364) dirty exists 2024-09-14T03:41:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:41:41] "GET /metrics HTTP/1.1" 200 36132 "" "Prometheus/2.51.0" 2024-09-14T03:41:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[86423]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:41:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[86423]: Reconfiguring daemon osd.7 on smithi195 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[86423]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[88262]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[88262]: Reconfiguring daemon osd.7 on smithi195 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[88262]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:41.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:41.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:41 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:41.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:41 smithi195 ceph-mon[82388]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:41:41.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:41 smithi195 ceph-mon[82388]: Reconfiguring daemon osd.7 on smithi195 2024-09-14T03:41:41.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:41 smithi195 ceph-mon[82388]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:41:41.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:41 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:41 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:41.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:41 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:41.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:41 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:41:41.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:41 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:42.726 INFO:tasks.rados.rados.0.smithi110.stdout:1222: done (2 left) 2024-09-14T03:41:42.726 INFO:tasks.rados.rados.0.smithi110.stdout:1223: done (1 left) 2024-09-14T03:41:42.726 INFO:tasks.rados.rados.0.smithi110.stdout:1224: done (0 left) 2024-09-14T03:41:42.726 INFO:tasks.rados.rados.0.smithi110.stdout:1225: copy_from oid 29 from oid 50 current snap is 126 2024-09-14T03:41:42.726 INFO:tasks.rados.rados.0.smithi110.stdout:1226: snap_create 2024-09-14T03:41:42.742 INFO:tasks.rados.rados.0.smithi110.stdout:1225: finishing copy_from racing read to smithi11075562-29 2024-09-14T03:41:42.744 INFO:tasks.rados.rados.0.smithi110.stdout:1225: finishing copy_from to smithi11075562-29 2024-09-14T03:41:42.744 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 101 (ObjNum 352 snap 114 seq_num 352) dirty exists 2024-09-14T03:41:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:42 smithi195 ceph-mon[82388]: Reconfiguring mds.b (monmap changed)... 2024-09-14T03:41:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:42 smithi195 ceph-mon[82388]: Reconfiguring daemon mds.b on smithi195 2024-09-14T03:41:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:42 smithi195 ceph-mon[82388]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:41:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:42 smithi195 ceph-mon[82388]: pgmap v85: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:43.153 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[88262]: Reconfiguring mds.b (monmap changed)... 2024-09-14T03:41:43.153 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[88262]: Reconfiguring daemon mds.b on smithi195 2024-09-14T03:41:43.153 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[88262]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:41:43.154 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[88262]: pgmap v85: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:43.154 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:43.154 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[86423]: Reconfiguring mds.b (monmap changed)... 2024-09-14T03:41:43.154 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[86423]: Reconfiguring daemon mds.b on smithi195 2024-09-14T03:41:43.154 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[86423]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:41:43.154 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[86423]: pgmap v85: 121 pgs: 121 active+clean; 297 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:43.154 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:43.673 INFO:tasks.rados.rados.0.smithi110.stdout:1225: done (1 left) 2024-09-14T03:41:43.674 INFO:tasks.rados.rados.0.smithi110.stdout:1226: done (0 left) 2024-09-14T03:41:43.674 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 127 2024-09-14T03:41:43.674 INFO:tasks.rados.rados.0.smithi110.stdout:1227: seq_num 386 ranges {3399680=466944,3866624=638976,4505600=565248} 2024-09-14T03:41:43.693 INFO:tasks.rados.rados.0.smithi110.stdout:1227: writing smithi11075562-28 from 3399680 to 3866624 tid 1 2024-09-14T03:41:43.697 INFO:tasks.rados.rados.0.smithi110.stdout:1227: writing smithi11075562-28 from 3866624 to 4505600 tid 2 2024-09-14T03:41:43.701 INFO:tasks.rados.rados.0.smithi110.stdout:1227: writing smithi11075562-28 from 4505600 to 5070848 tid 3 2024-09-14T03:41:43.701 INFO:tasks.rados.rados.0.smithi110.stdout:1228: rmattr oid 39 current snap is 127 2024-09-14T03:41:43.701 INFO:tasks.rados.rados.0.smithi110.stdout:1228: done (1 left) 2024-09-14T03:41:43.701 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 34 current snap is 127 2024-09-14T03:41:43.701 INFO:tasks.rados.rados.0.smithi110.stdout:1227: finishing write tid 1 to smithi11075562-28 2024-09-14T03:41:43.701 INFO:tasks.rados.rados.0.smithi110.stdout:1227: finishing write tid 2 to smithi11075562-28 2024-09-14T03:41:43.701 INFO:tasks.rados.rados.0.smithi110.stdout:1229: seq_num 387 ranges {2498560=483328,2981888=524288,3506176=679936} 2024-09-14T03:41:43.717 INFO:tasks.rados.rados.0.smithi110.stdout:1229: writing smithi11075562-34 from 2498560 to 2981888 tid 1 2024-09-14T03:41:43.720 INFO:tasks.rados.rados.0.smithi110.stdout:1229: writing smithi11075562-34 from 2981888 to 3506176 tid 2 2024-09-14T03:41:43.725 INFO:tasks.rados.rados.0.smithi110.stdout:1229: writing smithi11075562-34 from 3506176 to 4186112 tid 3 2024-09-14T03:41:43.725 INFO:tasks.rados.rados.0.smithi110.stdout:1230: copy_from oid 3 from oid 13 current snap is 127 2024-09-14T03:41:43.725 INFO:tasks.rados.rados.0.smithi110.stdout:1227: finishing write tid 3 to smithi11075562-28 2024-09-14T03:41:43.725 INFO:tasks.rados.rados.0.smithi110.stdout:1227: finishing write tid 4 to smithi11075562-28 2024-09-14T03:41:43.726 INFO:tasks.rados.rados.0.smithi110.stdout:1227: finishing write tid 5 to smithi11075562-28 2024-09-14T03:41:43.726 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 454 (ObjNum 386 snap 127 seq_num 386) dirty exists 2024-09-14T03:41:43.726 INFO:tasks.rados.rados.0.smithi110.stdout:1227: left oid 28 (ObjNum 386 snap 127 seq_num 386) 2024-09-14T03:41:43.726 INFO:tasks.rados.rados.0.smithi110.stdout:1229: finishing write tid 1 to smithi11075562-34 2024-09-14T03:41:43.726 INFO:tasks.rados.rados.0.smithi110.stdout:1227: done (2 left) 2024-09-14T03:41:43.726 INFO:tasks.rados.rados.0.smithi110.stdout:1231: copy_from oid 5 from oid 45 current snap is 127 2024-09-14T03:41:43.726 INFO:tasks.rados.rados.0.smithi110.stdout:1232: snap_create 2024-09-14T03:41:43.727 INFO:tasks.rados.rados.0.smithi110.stdout:1229: finishing write tid 2 to smithi11075562-34 2024-09-14T03:41:43.730 INFO:tasks.rados.rados.0.smithi110.stdout:1229: finishing write tid 3 to smithi11075562-34 2024-09-14T03:41:43.730 INFO:tasks.rados.rados.0.smithi110.stdout:1229: finishing write tid 4 to smithi11075562-34 2024-09-14T03:41:43.731 INFO:tasks.rados.rados.0.smithi110.stdout:1229: finishing write tid 5 to smithi11075562-34 2024-09-14T03:41:43.731 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 498 (ObjNum 387 snap 127 seq_num 387) dirty exists 2024-09-14T03:41:43.731 INFO:tasks.rados.rados.0.smithi110.stdout:1229: left oid 34 (ObjNum 387 snap 127 seq_num 387) 2024-09-14T03:41:43.733 INFO:tasks.rados.rados.0.smithi110.stdout:1231: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:41:43.736 INFO:tasks.rados.rados.0.smithi110.stdout:1231: finishing copy_from to smithi11075562-5 2024-09-14T03:41:43.736 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 787 (ObjNum 376 snap 123 seq_num 376) dirty exists 2024-09-14T03:41:43.763 INFO:tasks.rados.rados.0.smithi110.stdout:1230: finishing copy_from racing read to smithi11075562-3 2024-09-14T03:41:43.766 INFO:tasks.rados.rados.0.smithi110.stdout:1230: finishing copy_from to smithi11075562-3 2024-09-14T03:41:43.766 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 662 (ObjNum 356 snap 115 seq_num 356) dirty exists 2024-09-14T03:41:43.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:41:43.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:43.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:43.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:41:43.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[86423]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:43.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:41:43.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:41:43.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:43.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:43.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:41:43.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:43 smithi110 ceph-mon[88262]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:41:44.117 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:41:44.118 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:41:44.119 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:41:44.119 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:41:44.119 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:41:44.119 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:44.119 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:44.119 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:41:44.119 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:43 smithi195 ceph-mon[82388]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:41:44.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:44.730 INFO:tasks.rados.rados.0.smithi110.stdout:1229: done (3 left) 2024-09-14T03:41:44.730 INFO:tasks.rados.rados.0.smithi110.stdout:1230: done (2 left) 2024-09-14T03:41:44.730 INFO:tasks.rados.rados.0.smithi110.stdout:1231: done (1 left) 2024-09-14T03:41:44.730 INFO:tasks.rados.rados.0.smithi110.stdout:1232: done (0 left) 2024-09-14T03:41:44.730 INFO:tasks.rados.rados.0.smithi110.stdout:1233: snap_create 2024-09-14T03:41:44.880 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all mon 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all crash 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[88262]: Upgrade: osd.0 is safe to restart 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[88262]: pgmap v88: 121 pgs: 121 active+clean; 299 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 776 KiB/s wr, 22 op/s 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all mon 2024-09-14T03:41:44.881 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all crash 2024-09-14T03:41:44.882 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:41:44.882 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[86423]: Upgrade: osd.0 is safe to restart 2024-09-14T03:41:44.882 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[86423]: pgmap v88: 121 pgs: 121 active+clean; 299 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 776 KiB/s wr, 22 op/s 2024-09-14T03:41:44.882 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:44.882 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:41:44.882 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:44 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:45.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all mon 2024-09-14T03:41:45.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all crash 2024-09-14T03:41:45.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:41:45.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-mon[82388]: Upgrade: osd.0 is safe to restart 2024-09-14T03:41:45.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-mon[82388]: pgmap v88: 121 pgs: 121 active+clean; 299 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 776 KiB/s wr, 22 op/s 2024-09-14T03:41:45.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:45.079 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:41:45.079 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:44 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:41:45.735 INFO:tasks.rados.rados.0.smithi110.stdout:1233: done (0 left) 2024-09-14T03:41:45.735 INFO:tasks.rados.rados.0.smithi110.stdout:1234: snap_remove snap 128 2024-09-14T03:41:45.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:45 smithi110 ceph-mon[88262]: Upgrade: Updating osd.0 2024-09-14T03:41:45.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:45 smithi110 ceph-mon[88262]: Deploying daemon osd.0 on smithi110 2024-09-14T03:41:45.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:45 smithi110 ceph-mon[88262]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:41:45.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:45 smithi110 ceph-mon[86423]: Upgrade: Updating osd.0 2024-09-14T03:41:45.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:45 smithi110 ceph-mon[86423]: Deploying daemon osd.0 on smithi110 2024-09-14T03:41:45.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:45 smithi110 ceph-mon[86423]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:41:46.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:45 smithi195 ceph-mon[82388]: Upgrade: Updating osd.0 2024-09-14T03:41:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:45 smithi195 ceph-mon[82388]: Deploying daemon osd.0 on smithi110 2024-09-14T03:41:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:45 smithi195 ceph-mon[82388]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:41:46.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:46.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:46.735 INFO:tasks.rados.rados.0.smithi110.stdout:1234: done (0 left) 2024-09-14T03:41:46.736 INFO:tasks.rados.rados.0.smithi110.stdout:1235: rollback oid 5 current snap is 129 2024-09-14T03:41:46.736 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 5 to 123 2024-09-14T03:41:46.736 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 129 2024-09-14T03:41:46.736 INFO:tasks.rados.rados.0.smithi110.stdout:1236: seq_num 388 ranges {2392064=507904,2899968=458752,3358720=606208,3964928=139264} 2024-09-14T03:41:46.752 INFO:tasks.rados.rados.0.smithi110.stdout:1236: writing smithi11075562-29 from 2392064 to 2899968 tid 1 2024-09-14T03:41:46.755 INFO:tasks.rados.rados.0.smithi110.stdout:1236: writing smithi11075562-29 from 2899968 to 3358720 tid 2 2024-09-14T03:41:46.760 INFO:tasks.rados.rados.0.smithi110.stdout:1236: writing smithi11075562-29 from 3358720 to 3964928 tid 3 2024-09-14T03:41:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:1236: writing smithi11075562-29 from 3964928 to 4104192 tid 4 2024-09-14T03:41:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 129 2024-09-14T03:41:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:1235: finishing rollback tid 0 to smithi11075562-5 2024-09-14T03:41:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:1235: finishing rollback tid 1 to smithi11075562-5 2024-09-14T03:41:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:1235: finishing rollback tid 2 to smithi11075562-5 2024-09-14T03:41:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 829 (ObjNum 365 snap 118 seq_num 365) dirty exists 2024-09-14T03:41:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:1236: finishing write tid 1 to smithi11075562-29 2024-09-14T03:41:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:1236: finishing write tid 2 to smithi11075562-29 2024-09-14T03:41:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:1237: seq_num 389 ranges {5079040=524288,5603328=745472,6348800=458752} 2024-09-14T03:41:46.793 INFO:tasks.rados.rados.0.smithi110.stdout:1237: writing smithi11075562-10 from 5079040 to 5603328 tid 1 2024-09-14T03:41:46.798 INFO:tasks.rados.rados.0.smithi110.stdout:1237: writing smithi11075562-10 from 5603328 to 6348800 tid 2 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:1237: writing smithi11075562-10 from 6348800 to 6807552 tid 3 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:1235: done (2 left) 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:1238: delete oid 17 current snap is 129 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:1236: finishing write tid 3 to smithi11075562-29 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:1236: finishing write tid 4 to smithi11075562-29 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:1236: finishing write tid 5 to smithi11075562-29 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:1236: finishing write tid 6 to smithi11075562-29 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 339 (ObjNum 388 snap 129 seq_num 388) dirty exists 2024-09-14T03:41:46.802 INFO:tasks.rados.rados.0.smithi110.stdout:1236: left oid 29 (ObjNum 388 snap 129 seq_num 388) 2024-09-14T03:41:46.803 INFO:tasks.rados.rados.0.smithi110.stdout:1237: finishing write tid 1 to smithi11075562-10 2024-09-14T03:41:46.803 INFO:tasks.rados.rados.0.smithi110.stdout:1237: finishing write tid 2 to smithi11075562-10 2024-09-14T03:41:46.804 INFO:tasks.rados.rados.0.smithi110.stdout:1237: finishing write tid 3 to smithi11075562-10 2024-09-14T03:41:46.804 INFO:tasks.rados.rados.0.smithi110.stdout:1236: done (2 left) 2024-09-14T03:41:46.804 INFO:tasks.rados.rados.0.smithi110.stdout:1238: done (1 left) 2024-09-14T03:41:46.804 INFO:tasks.rados.rados.0.smithi110.stdout:1239: read oid 39 snap -1 2024-09-14T03:41:46.804 INFO:tasks.rados.rados.0.smithi110.stdout:1239: expect deleted 2024-09-14T03:41:46.804 INFO:tasks.rados.rados.0.smithi110.stdout:1240: snap_remove snap 109 2024-09-14T03:41:46.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:46 smithi110 ceph-mon[88262]: pgmap v90: 121 pgs: 121 active+clean; 299 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 774 KiB/s wr, 22 op/s 2024-09-14T03:41:46.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:46 smithi110 ceph-mon[88262]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:41:46.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:46 smithi110 ceph-mon[86423]: pgmap v90: 121 pgs: 121 active+clean; 299 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 774 KiB/s wr, 22 op/s 2024-09-14T03:41:46.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:46 smithi110 ceph-mon[86423]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:41:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:46 smithi195 ceph-mon[82388]: pgmap v90: 121 pgs: 121 active+clean; 299 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 774 KiB/s wr, 22 op/s 2024-09-14T03:41:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:46 smithi195 ceph-mon[82388]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:41:47.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:47.531 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:47 smithi110 systemd[1]: Stopping Ceph osd.0 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:41:47.748 INFO:tasks.rados.rados.0.smithi110.stdout:1240: done (2 left) 2024-09-14T03:41:47.748 INFO:tasks.rados.rados.0.smithi110.stdout:1241: snap_remove snap 108 2024-09-14T03:41:47.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:47 smithi110 ceph-mon[86423]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:41:47.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:47 smithi110 ceph-mon[86423]: osd.0 marked itself down and dead 2024-09-14T03:41:47.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:47 smithi110 ceph-mon[88262]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:41:47.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:47 smithi110 ceph-mon[88262]: osd.0 marked itself down and dead 2024-09-14T03:41:47.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:47 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[39236]: 2024-09-14T03:41:47.529+0000 7ff363696640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:41:47.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:47 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[39236]: 2024-09-14T03:41:47.529+0000 7ff363696640 -1 osd.0 311 *** Got signal Terminated *** 2024-09-14T03:41:47.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:47 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[39236]: 2024-09-14T03:41:47.529+0000 7ff363696640 -1 osd.0 311 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:41:48.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:47 smithi195 ceph-mon[82388]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:41:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:47 smithi195 ceph-mon[82388]: osd.0 marked itself down and dead 2024-09-14T03:41:48.574 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:48 smithi110 podman[92979]: 2024-09-14 03:41:48.296253043 +0000 UTC m=+0.877369422 container died 7a3ad928b263fbb5c05050d5146cbb7a4d8d59e0a9afd234994f87754e2346e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T03:41:48.679 INFO:tasks.rados.rados.0.smithi110.stdout:1241: done (2 left) 2024-09-14T03:41:48.679 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 129 2024-09-14T03:41:48.679 INFO:tasks.rados.rados.0.smithi110.stdout:1242: seq_num 391 ranges {1556480=557056,2113536=516096,2629632=688128} 2024-09-14T03:41:48.690 INFO:tasks.rados.rados.0.smithi110.stdout:1242: writing smithi11075562-47 from 1556480 to 2113536 tid 1 2024-09-14T03:41:48.694 INFO:tasks.rados.rados.0.smithi110.stdout:1242: writing smithi11075562-47 from 2113536 to 2629632 tid 2 2024-09-14T03:41:48.699 INFO:tasks.rados.rados.0.smithi110.stdout:1242: writing smithi11075562-47 from 2629632 to 3317760 tid 3 2024-09-14T03:41:48.699 INFO:tasks.rados.rados.0.smithi110.stdout:1243: copy_from oid 46 from oid 44 current snap is 129 2024-09-14T03:41:48.699 INFO:tasks.rados.rados.0.smithi110.stdout:1237: finishing write tid 4 to smithi11075562-10 2024-09-14T03:41:48.699 INFO:tasks.rados.rados.0.smithi110.stdout:1237: finishing write tid 5 to smithi11075562-10 2024-09-14T03:41:48.699 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 502 (ObjNum 389 snap 129 seq_num 389) dirty exists 2024-09-14T03:41:48.700 INFO:tasks.rados.rados.0.smithi110.stdout:1237: left oid 10 (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:41:48.700 INFO:tasks.rados.rados.0.smithi110.stdout:1242: finishing write tid 1 to smithi11075562-47 2024-09-14T03:41:48.700 INFO:tasks.rados.rados.0.smithi110.stdout:1237: done (3 left) 2024-09-14T03:41:48.700 INFO:tasks.rados.rados.0.smithi110.stdout:1239: done (2 left) 2024-09-14T03:41:48.700 INFO:tasks.rados.rados.0.smithi110.stdout:1244: snap_create 2024-09-14T03:41:48.700 INFO:tasks.rados.rados.0.smithi110.stdout:1242: finishing write tid 2 to smithi11075562-47 2024-09-14T03:41:48.704 INFO:tasks.rados.rados.0.smithi110.stdout:1242: finishing write tid 3 to smithi11075562-47 2024-09-14T03:41:48.704 INFO:tasks.rados.rados.0.smithi110.stdout:1242: finishing write tid 4 to smithi11075562-47 2024-09-14T03:41:48.705 INFO:tasks.rados.rados.0.smithi110.stdout:1242: finishing write tid 5 to smithi11075562-47 2024-09-14T03:41:48.705 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 840 (ObjNum 391 snap 129 seq_num 391) dirty exists 2024-09-14T03:41:48.705 INFO:tasks.rados.rados.0.smithi110.stdout:1242: left oid 47 (ObjNum 391 snap 129 seq_num 391) 2024-09-14T03:41:48.714 INFO:tasks.rados.rados.0.smithi110.stdout:1243: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:41:48.716 INFO:tasks.rados.rados.0.smithi110.stdout:1243: finishing copy_from to smithi11075562-46 2024-09-14T03:41:48.716 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 381 (ObjNum 313 snap 100 seq_num 313) dirty exists 2024-09-14T03:41:48.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:48 smithi110 ceph-mon[86423]: pgmap v93: 121 pgs: 121 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1010 B/s rd, 4.5 MiB/s wr, 7 op/s 2024-09-14T03:41:48.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:48 smithi110 ceph-mon[86423]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:41:48.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:48 smithi110 ceph-mon[86423]: osdmap e312: 8 total, 7 up, 8 in 2024-09-14T03:41:48.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:48 smithi110 ceph-mon[86423]: osdmap e313: 8 total, 7 up, 8 in 2024-09-14T03:41:48.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:48 smithi110 ceph-mon[88262]: pgmap v93: 121 pgs: 121 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1010 B/s rd, 4.5 MiB/s wr, 7 op/s 2024-09-14T03:41:48.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:48 smithi110 ceph-mon[88262]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:41:48.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:48 smithi110 ceph-mon[88262]: osdmap e312: 8 total, 7 up, 8 in 2024-09-14T03:41:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:48 smithi110 ceph-mon[88262]: osdmap e313: 8 total, 7 up, 8 in 2024-09-14T03:41:48.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:48 smithi110 podman[92979]: 2024-09-14 03:41:48.671413864 +0000 UTC m=+1.252530244 container cleanup 7a3ad928b263fbb5c05050d5146cbb7a4d8d59e0a9afd234994f87754e2346e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T03:41:48.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:48 smithi110 bash[92979]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0 2024-09-14T03:41:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:48 smithi195 ceph-mon[82388]: pgmap v93: 121 pgs: 121 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1010 B/s rd, 4.5 MiB/s wr, 7 op/s 2024-09-14T03:41:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:48 smithi195 ceph-mon[82388]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:41:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:48 smithi195 ceph-mon[82388]: osdmap e312: 8 total, 7 up, 8 in 2024-09-14T03:41:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:48 smithi195 ceph-mon[82388]: osdmap e313: 8 total, 7 up, 8 in 2024-09-14T03:41:49.349 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:49 smithi110 podman[92995]: 2024-09-14 03:41:49.021943052 +0000 UTC m=+0.721138151 container remove 7a3ad928b263fbb5c05050d5146cbb7a4d8d59e0a9afd234994f87754e2346e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=quincy-eb16b97) 2024-09-14T03:41:49.352 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:41:49.677 INFO:tasks.rados.rados.0.smithi110.stdout:1242: done (2 left) 2024-09-14T03:41:49.677 INFO:tasks.rados.rados.0.smithi110.stdout:1243: done (1 left) 2024-09-14T03:41:49.677 INFO:tasks.rados.rados.0.smithi110.stdout:1244: done (0 left) 2024-09-14T03:41:49.677 INFO:tasks.rados.rados.0.smithi110.stdout:1245: delete oid 3 current snap is 130 2024-09-14T03:41:49.678 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:49 smithi110 podman[93106]: 2024-09-14 03:41:49.251017162 +0000 UTC m=+0.016020660 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:41:49.678 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:49 smithi110 podman[93106]: 2024-09-14 03:41:49.37427371 +0000 UTC m=+0.139277206 container create d78133aa30ccfc8e21be6ab1b636a6c3c3282ea95ba4b572f91ecb0168ce66ef (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-a278231, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T03:41:49.678 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:49 smithi110 podman[93106]: 2024-09-14 03:41:49.593732183 +0000 UTC m=+0.358735682 container init d78133aa30ccfc8e21be6ab1b636a6c3c3282ea95ba4b572f91ecb0168ce66ef (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:41:49.678 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:49 smithi110 podman[93106]: 2024-09-14 03:41:49.597202567 +0000 UTC m=+0.362206076 container start d78133aa30ccfc8e21be6ab1b636a6c3c3282ea95ba4b572f91ecb0168ce66ef (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=) 2024-09-14T03:41:49.678 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:49 smithi110 podman[93106]: 2024-09-14 03:41:49.67707048 +0000 UTC m=+0.442074005 container attach d78133aa30ccfc8e21be6ab1b636a6c3c3282ea95ba4b572f91ecb0168ce66ef (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True) 2024-09-14T03:41:49.683 INFO:tasks.rados.rados.0.smithi110.stdout:1245: done (0 left) 2024-09-14T03:41:49.683 INFO:tasks.rados.rados.0.smithi110.stdout:1246: read oid 6 snap -1 2024-09-14T03:41:49.684 INFO:tasks.rados.rados.0.smithi110.stdout:1246: expect (ObjNum 244 snap 69 seq_num 244) 2024-09-14T03:41:49.684 INFO:tasks.rados.rados.0.smithi110.stdout:1247: delete oid 30 current snap is 130 2024-09-14T03:41:49.687 INFO:tasks.rados.rados.0.smithi110.stdout:1247: done (1 left) 2024-09-14T03:41:49.687 INFO:tasks.rados.rados.0.smithi110.stdout:1248: snap_remove snap 124 2024-09-14T03:41:49.937 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:41:49.937 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (2m) 45s ago 6m 16.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:41:49.937 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (2m) 31s ago 6m 78.7M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:41:49.937 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (8m) 45s ago 8m 18.9M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:41:49.937 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (8m) 31s ago 8m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:41:49.937 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (90s) 31s ago 11m 452M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:41:49.937 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (117s) 45s ago 12m 516M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (76s) 45s ago 13m 39.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (36s) 31s ago 11m 21.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (54s) 45s ago 11m 27.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (2m) 45s ago 6m 9697k - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (2m) 31s ago 6m 9722k - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (11m) 45s ago 11m 169M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 7a3ad928b263 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (10m) 45s ago 10m 163M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (10m) 45s ago 10m 127M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (9m) 45s ago 9m 119M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (9m) 31s ago 9m 229M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (9m) 31s ago 9m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (8m) 31s ago 8m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (8m) 31s ago 8m 106M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:41:49.938 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (95s) 31s ago 7m 44.2M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:41:49.939 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:49 smithi110 podman[93106]: 2024-09-14 03:41:49.809331116 +0000 UTC m=+0.574334619 container died d78133aa30ccfc8e21be6ab1b636a6c3c3282ea95ba4b572f91ecb0168ce66ef (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-a278231, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-14T03:41:50.278 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:41:50.278 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 7 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:41:50.279 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 9, 2024-09-14T03:41:50.280 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T03:41:50.280 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:41:50.280 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:41:50.573 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:50 smithi110 podman[93175]: 2024-09-14 03:41:50.423384516 +0000 UTC m=+0.604833656 container remove d78133aa30ccfc8e21be6ab1b636a6c3c3282ea95ba4b572f91ecb0168ce66ef (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:41:50.574 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:50 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-14T03:41:50.574 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:50 smithi110 systemd[1]: Stopped Ceph osd.0 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:41:50.574 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:50 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.0.service: Consumed 10.724s CPU time. 2024-09-14T03:41:50.574 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:41:50.574 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:41:50.574 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:41:50.574 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:41:50.574 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:41:50.574 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:41:50.575 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:41:50.575 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:41:50.575 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "5/20 daemons upgraded", 2024-09-14T03:41:50.575 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:41:50.575 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:41:50.575 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:41:50.679 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:50.688 INFO:tasks.rados.rados.0.smithi110.stdout:1248: done (1 left) 2024-09-14T03:41:50.688 INFO:tasks.rados.rados.0.smithi110.stdout:1249: read oid 44 snap -1 2024-09-14T03:41:50.688 INFO:tasks.rados.rados.0.smithi110.stdout:1249: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:50.689 INFO:tasks.rados.rados.0.smithi110.stdout:1250: delete oid 5 current snap is 130 2024-09-14T03:41:50.689 INFO:tasks.rados.rados.0.smithi110.stdout:1246: expect (ObjNum 244 snap 69 seq_num 244) 2024-09-14T03:41:50.860 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:50 smithi110 systemd[1]: Starting Ceph osd.0 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:41:50.861 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[88262]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:50.861 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[88262]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:50.861 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[88262]: osdmap e314: 8 total, 7 up, 8 in 2024-09-14T03:41:50.861 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[88262]: pgmap v97: 121 pgs: 15 stale+active+clean, 106 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.0 KiB/s rd, 4.6 MiB/s wr, 7 op/s 2024-09-14T03:41:50.861 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2755660183' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:50.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[86423]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:50.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[86423]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:50.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[86423]: osdmap e314: 8 total, 7 up, 8 in 2024-09-14T03:41:50.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[86423]: pgmap v97: 121 pgs: 15 stale+active+clean, 106 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.0 KiB/s rd, 4.6 MiB/s wr, 7 op/s 2024-09-14T03:41:50.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:50 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2755660183' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:50 smithi195 ceph-mon[82388]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:50 smithi195 ceph-mon[82388]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:50 smithi195 ceph-mon[82388]: osdmap e314: 8 total, 7 up, 8 in 2024-09-14T03:41:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:50 smithi195 ceph-mon[82388]: pgmap v97: 121 pgs: 15 stale+active+clean, 106 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.0 KiB/s rd, 4.6 MiB/s wr, 7 op/s 2024-09-14T03:41:50.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:50 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2755660183' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:41:50.997 INFO:tasks.rados.rados.0.smithi110.stdout:1249: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:51.178 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:50 smithi110 podman[93319]: 2024-09-14 03:41:50.761929338 +0000 UTC m=+0.016766788 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:41:51.178 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:50 smithi110 podman[93319]: 2024-09-14 03:41:50.86610316 +0000 UTC m=+0.120940599 container create b3270cf817ed2d954c7f7295d661900696b5c6d661b29d53e57e51434e23b43c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T03:41:51.178 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 podman[93319]: 2024-09-14 03:41:51.068795843 +0000 UTC m=+0.323633288 container init b3270cf817ed2d954c7f7295d661900696b5c6d661b29d53e57e51434e23b43c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-14T03:41:51.179 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 podman[93319]: 2024-09-14 03:41:51.072128319 +0000 UTC m=+0.326965759 container start b3270cf817ed2d954c7f7295d661900696b5c6d661b29d53e57e51434e23b43c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-14T03:41:51.179 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 podman[93319]: 2024-09-14 03:41:51.177547245 +0000 UTC m=+0.432384705 container attach b3270cf817ed2d954c7f7295d661900696b5c6d661b29d53e57e51434e23b43c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T03:41:51.200 INFO:tasks.rados.rados.0.smithi110.stdout:1246: done (2 left) 2024-09-14T03:41:51.200 INFO:tasks.rados.rados.0.smithi110.stdout:1249: done (1 left) 2024-09-14T03:41:51.200 INFO:tasks.rados.rados.0.smithi110.stdout:1250: done (0 left) 2024-09-14T03:41:51.201 INFO:tasks.rados.rados.0.smithi110.stdout:1251: delete oid 13 current snap is 130 2024-09-14T03:41:51.205 INFO:tasks.rados.rados.0.smithi110.stdout:1251: done (0 left) 2024-09-14T03:41:51.205 INFO:tasks.rados.rados.0.smithi110.stdout:1252: read oid 28 snap -1 2024-09-14T03:41:51.205 INFO:tasks.rados.rados.0.smithi110.stdout:1252: expect (ObjNum 386 snap 127 seq_num 386) 2024-09-14T03:41:51.205 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 130 2024-09-14T03:41:51.205 INFO:tasks.rados.rados.0.smithi110.stdout:1253: seq_num 396 ranges {0=638976,638976=786432,1425408=417792} 2024-09-14T03:41:51.209 INFO:tasks.rados.rados.0.smithi110.stdout:1253: writing smithi11075562-30 from 0 to 638976 tid 1 2024-09-14T03:41:51.215 INFO:tasks.rados.rados.0.smithi110.stdout:1253: writing smithi11075562-30 from 638976 to 1425408 tid 2 2024-09-14T03:41:51.219 INFO:tasks.rados.rados.0.smithi110.stdout:1253: writing smithi11075562-30 from 1425408 to 1843200 tid 3 2024-09-14T03:41:51.219 INFO:tasks.rados.rados.0.smithi110.stdout:1254: read oid 24 snap -1 2024-09-14T03:41:51.220 INFO:tasks.rados.rados.0.smithi110.stdout:1254: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:51.220 INFO:tasks.rados.rados.0.smithi110.stdout:1253: finishing write tid 1 to smithi11075562-30 2024-09-14T03:41:51.220 INFO:tasks.rados.rados.0.smithi110.stdout:1255: read oid 19 snap 103 2024-09-14T03:41:51.220 INFO:tasks.rados.rados.0.smithi110.stdout:1255: expect (ObjNum 235 snap 69 seq_num 235) 2024-09-14T03:41:51.220 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 130 2024-09-14T03:41:51.220 INFO:tasks.rados.rados.0.smithi110.stdout:1256: seq_num 397 ranges {0=655360,655360=671744,1327104=532480} 2024-09-14T03:41:51.224 INFO:tasks.rados.rados.0.smithi110.stdout:1256: writing smithi11075562-18 from 0 to 655360 tid 1 2024-09-14T03:41:51.229 INFO:tasks.rados.rados.0.smithi110.stdout:1256: writing smithi11075562-18 from 655360 to 1327104 tid 2 2024-09-14T03:41:51.234 INFO:tasks.rados.rados.0.smithi110.stdout:1256: writing smithi11075562-18 from 1327104 to 1859584 tid 3 2024-09-14T03:41:51.234 INFO:tasks.rados.rados.0.smithi110.stdout:1257: read oid 32 snap -1 2024-09-14T03:41:51.234 INFO:tasks.rados.rados.0.smithi110.stdout:1257: expect (ObjNum 211 snap 65 seq_num 211) 2024-09-14T03:41:51.234 INFO:tasks.rados.rados.0.smithi110.stdout:1258: delete oid 39 current snap is 130 2024-09-14T03:41:51.234 INFO:tasks.rados.rados.0.smithi110.stdout:1253: finishing write tid 2 to smithi11075562-30 2024-09-14T03:41:51.234 INFO:tasks.rados.rados.0.smithi110.stdout:1253: finishing write tid 3 to smithi11075562-30 2024-09-14T03:41:51.234 INFO:tasks.rados.rados.0.smithi110.stdout:1253: finishing write tid 4 to smithi11075562-30 2024-09-14T03:41:51.234 INFO:tasks.rados.rados.0.smithi110.stdout:1253: finishing write tid 5 to smithi11075562-30 2024-09-14T03:41:51.235 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 730 (ObjNum 396 snap 130 seq_num 396) dirty exists 2024-09-14T03:41:51.235 INFO:tasks.rados.rados.0.smithi110.stdout:1253: left oid 30 (ObjNum 396 snap 130 seq_num 396) 2024-09-14T03:41:51.235 INFO:tasks.rados.rados.0.smithi110.stdout:1252: expect (ObjNum 386 snap 127 seq_num 386) 2024-09-14T03:41:51.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:41:51.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:41:51.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:41:51.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:41:51.511 INFO:tasks.rados.rados.0.smithi110.stdout:1255: expect (ObjNum 235 snap 69 seq_num 235) 2024-09-14T03:41:51.550 INFO:tasks.rados.rados.0.smithi110.stdout:1256: finishing write tid 1 to smithi11075562-18 2024-09-14T03:41:51.550 INFO:tasks.rados.rados.0.smithi110.stdout:1256: finishing write tid 2 to smithi11075562-18 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1256: finishing write tid 3 to smithi11075562-18 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1252: done (6 left) 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1253: done (5 left) 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1255: done (4 left) 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1258: done (3 left) 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1259: snap_create 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1256: finishing write tid 4 to smithi11075562-18 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1256: finishing write tid 5 to smithi11075562-18 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 627 (ObjNum 397 snap 130 seq_num 397) dirty exists 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1256: left oid 18 (ObjNum 397 snap 130 seq_num 397) 2024-09-14T03:41:51.551 INFO:tasks.rados.rados.0.smithi110.stdout:1254: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:51.746 INFO:tasks.rados.rados.0.smithi110.stdout:1257: expect (ObjNum 211 snap 65 seq_num 211) 2024-09-14T03:41:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:41:51] "GET /metrics HTTP/1.1" 200 36134 "" "Prometheus/2.51.0" 2024-09-14T03:41:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-mon[86423]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-mon[86423]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-mon[86423]: osdmap e315: 8 total, 7 up, 8 in 2024-09-14T03:41:51.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-mon[88262]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:51.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-mon[88262]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:51.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-mon[88262]: osdmap e315: 8 total, 7 up, 8 in 2024-09-14T03:41:51.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:41:51.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:41:51.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:41:51.947 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:41:51.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:51 smithi195 ceph-mon[82388]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:51.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:51 smithi195 ceph-mon[82388]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:51.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:51 smithi195 ceph-mon[82388]: osdmap e315: 8 total, 7 up, 8 in 2024-09-14T03:41:51.959 INFO:tasks.rados.rados.0.smithi110.stdout:1254: done (3 left) 2024-09-14T03:41:51.959 INFO:tasks.rados.rados.0.smithi110.stdout:1256: done (2 left) 2024-09-14T03:41:51.959 INFO:tasks.rados.rados.0.smithi110.stdout:1257: done (1 left) 2024-09-14T03:41:51.960 INFO:tasks.rados.rados.0.smithi110.stdout:1259: done (0 left) 2024-09-14T03:41:51.960 INFO:tasks.rados.rados.0.smithi110.stdout:1260: copy_from oid 38 from oid 8 current snap is 131 2024-09-14T03:41:51.960 INFO:tasks.rados.rados.0.smithi110.stdout:1261: delete oid 42 current snap is 131 2024-09-14T03:41:51.962 INFO:tasks.rados.rados.0.smithi110.stdout:1261: done (1 left) 2024-09-14T03:41:51.962 INFO:tasks.rados.rados.0.smithi110.stdout:1262: delete oid 29 current snap is 131 2024-09-14T03:41:51.963 INFO:tasks.rados.rados.0.smithi110.stdout:1262: done (1 left) 2024-09-14T03:41:51.964 INFO:tasks.rados.rados.0.smithi110.stdout:1263: rollback oid 27 current snap is 131 2024-09-14T03:41:51.964 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 121 2024-09-14T03:41:51.964 INFO:tasks.rados.rados.0.smithi110.stdout:1264: snap_remove snap 125 2024-09-14T03:41:52.298 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:41:52.298 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:41:52.298 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:41:52.298 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:41:52.298 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T03:41:52.299 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T03:41:52.299 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:41:52.299 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:41:52.299 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate[93334]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T03:41:52.299 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:51 smithi110 bash[93319]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T03:41:52.299 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:52 smithi110 podman[93319]: 2024-09-14 03:41:52.012655795 +0000 UTC m=+1.267493239 container died b3270cf817ed2d954c7f7295d661900696b5c6d661b29d53e57e51434e23b43c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:1264: done (2 left) 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:1265: copy_from oid 3 from oid 44 current snap is 131 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:1263: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 108 (ObjNum 64603904 snap 21934 seq_num 304) dirty dne 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:1260: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:1260: finishing copy_from to smithi11075562-38 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 477 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:1260: done (2 left) 2024-09-14T03:41:52.694 INFO:tasks.rados.rados.0.smithi110.stdout:1263: done (1 left) 2024-09-14T03:41:52.695 INFO:tasks.rados.rados.0.smithi110.stdout:1266: rollback oid 23 current snap is 131 2024-09-14T03:41:52.695 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 23 to 123 2024-09-14T03:41:52.695 INFO:tasks.rados.rados.0.smithi110.stdout:1267: rollback oid 49 current snap is 131 2024-09-14T03:41:52.695 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 106 2024-09-14T03:41:52.695 INFO:tasks.rados.rados.0.smithi110.stdout:1268: snap_remove snap 122 2024-09-14T03:41:52.905 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:52 smithi110 ceph-mon[88262]: osdmap e316: 8 total, 7 up, 8 in 2024-09-14T03:41:52.905 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:52 smithi110 ceph-mon[88262]: pgmap v100: 121 pgs: 15 stale+active+clean, 106 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:52.906 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:52 smithi110 ceph-mon[86423]: osdmap e316: 8 total, 7 up, 8 in 2024-09-14T03:41:52.906 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:52 smithi110 ceph-mon[86423]: pgmap v100: 121 pgs: 15 stale+active+clean, 106 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:52.906 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:52 smithi110 podman[93476]: 2024-09-14 03:41:52.564994125 +0000 UTC m=+0.546645084 container remove b3270cf817ed2d954c7f7295d661900696b5c6d661b29d53e57e51434e23b43c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:41:52.906 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:52 smithi110 podman[93539]: 2024-09-14 03:41:52.671065206 +0000 UTC m=+0.015561903 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:41:52.906 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:52 smithi110 podman[93539]: 2024-09-14 03:41:52.783141379 +0000 UTC m=+0.127638060 container create 9626562f2155f97ad95c41a59b987d4502079930d0e84c7d276b20426e4a8fbf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T03:41:52.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:52 smithi195 ceph-mon[82388]: osdmap e316: 8 total, 7 up, 8 in 2024-09-14T03:41:52.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:52 smithi195 ceph-mon[82388]: pgmap v100: 121 pgs: 15 stale+active+clean, 106 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:41:53.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:53 smithi110 podman[93539]: 2024-09-14 03:41:53.002529528 +0000 UTC m=+0.347026222 container init 9626562f2155f97ad95c41a59b987d4502079930d0e84c7d276b20426e4a8fbf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T03:41:53.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:53 smithi110 podman[93539]: 2024-09-14 03:41:53.005932596 +0000 UTC m=+0.350429278 container start 9626562f2155f97ad95c41a59b987d4502079930d0e84c7d276b20426e4a8fbf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-14T03:41:53.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-osd[93559]: -- 172.21.15.110:0/539841435 <== mon.2 v2:172.21.15.110:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5568a40d1a40 con 0x5568a40afc00 2024-09-14T03:41:53.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:53 smithi110 bash[93539]: 9626562f2155f97ad95c41a59b987d4502079930d0e84c7d276b20426e4a8fbf 2024-09-14T03:41:53.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:53 smithi110 systemd[1]: Started Ceph osd.0 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:41:53.697 INFO:tasks.rados.rados.0.smithi110.stdout:1268: done (3 left) 2024-09-14T03:41:53.697 INFO:tasks.rados.rados.0.smithi110.stdout:1269: copy_from oid 19 from oid 11 current snap is 131 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1267: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1267: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 760 (ObjNum 129 snap 0 seq_num 64965776) dirty dne 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1267: done (3 left) 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1270: snap_create 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1266: finishing rollback tid 0 to smithi11075562-23 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1266: finishing rollback tid 1 to smithi11075562-23 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1266: finishing rollback tid 2 to smithi11075562-23 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 505 (ObjNum 371 snap 122 seq_num 371) dirty exists 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1265: finishing copy_from racing read to smithi11075562-3 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:1265: finishing copy_from to smithi11075562-3 2024-09-14T03:41:53.698 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 381 (ObjNum 313 snap 100 seq_num 313) dirty exists 2024-09-14T03:41:53.721 INFO:tasks.rados.rados.0.smithi110.stdout:1269: finishing copy_from racing read to smithi11075562-19 2024-09-14T03:41:53.723 INFO:tasks.rados.rados.0.smithi110.stdout:1269: finishing copy_from to smithi11075562-19 2024-09-14T03:41:53.724 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 480 (ObjNum 355 snap 115 seq_num 355) dirty exists 2024-09-14T03:41:53.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-mon[88262]: osdmap e317: 8 total, 7 up, 8 in 2024-09-14T03:41:53.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:53.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:53.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:53.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-mon[86423]: osdmap e317: 8 total, 7 up, 8 in 2024-09-14T03:41:53.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:53.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:53.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:53 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:53.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:53 smithi195 ceph-mon[82388]: osdmap e317: 8 total, 7 up, 8 in 2024-09-14T03:41:53.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:53 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:53.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:53 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:53.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:53 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:41:54.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:54.703 INFO:tasks.rados.rados.0.smithi110.stdout:1265: done (3 left) 2024-09-14T03:41:54.703 INFO:tasks.rados.rados.0.smithi110.stdout:1266: done (2 left) 2024-09-14T03:41:54.703 INFO:tasks.rados.rados.0.smithi110.stdout:1269: done (1 left) 2024-09-14T03:41:54.703 INFO:tasks.rados.rados.0.smithi110.stdout:1270: done (0 left) 2024-09-14T03:41:54.703 INFO:tasks.rados.rados.0.smithi110.stdout:1271: copy_from oid 35 from oid 22 current snap is 132 2024-09-14T03:41:54.703 INFO:tasks.rados.rados.0.smithi110.stdout:1272: rollback oid 20 current snap is 132 2024-09-14T03:41:54.703 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 117 2024-09-14T03:41:54.703 INFO:tasks.rados.rados.0.smithi110.stdout:1273: delete oid 27 current snap is 132 2024-09-14T03:41:54.707 INFO:tasks.rados.rados.0.smithi110.stdout:1273: done (2 left) 2024-09-14T03:41:54.707 INFO:tasks.rados.rados.0.smithi110.stdout:1274: rollback oid 36 current snap is 132 2024-09-14T03:41:54.707 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 36 to 106 2024-09-14T03:41:54.707 INFO:tasks.rados.rados.0.smithi110.stdout:1275: delete oid 7 current snap is 132 2024-09-14T03:41:54.709 INFO:tasks.rados.rados.0.smithi110.stdout:1272: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:41:54.709 INFO:tasks.rados.rados.0.smithi110.stdout:1272: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:41:54.710 INFO:tasks.rados.rados.0.smithi110.stdout:1275: done (3 left) 2024-09-14T03:41:54.710 INFO:tasks.rados.rados.0.smithi110.stdout:1276: copy_from oid 1 from oid 4 current snap is 132 2024-09-14T03:41:54.710 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 132 2024-09-14T03:41:54.710 INFO:tasks.rados.rados.0.smithi110.stdout:1277: seq_num 403 ranges {524288=753664,1277952=425984,1703936=720896,2424832=57344} 2024-09-14T03:41:54.721 INFO:tasks.rados.rados.0.smithi110.stdout:1277: writing smithi11075562-21 from 524288 to 1277952 tid 1 2024-09-14T03:41:54.721 INFO:tasks.rados.rados.0.smithi110.stdout:1277: writing smithi11075562-21 from 1277952 to 1703936 tid 2 2024-09-14T03:41:54.725 INFO:tasks.rados.rados.0.smithi110.stdout:1277: writing smithi11075562-21 from 1703936 to 2424832 tid 3 2024-09-14T03:41:54.726 INFO:tasks.rados.rados.0.smithi110.stdout:1277: writing smithi11075562-21 from 2424832 to 2482176 tid 4 2024-09-14T03:41:54.726 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 132 2024-09-14T03:41:54.726 INFO:tasks.rados.rados.0.smithi110.stdout:1272: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:41:54.726 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 507 (ObjNum 292 snap 90 seq_num 292) dirty exists 2024-09-14T03:41:54.727 INFO:tasks.rados.rados.0.smithi110.stdout:1274: finishing rollback tid 0 to smithi11075562-36 2024-09-14T03:41:54.727 INFO:tasks.rados.rados.0.smithi110.stdout:1278: seq_num 404 ranges {1843200=770048,2613248=712704,3325952=491520} 2024-09-14T03:41:54.740 INFO:tasks.rados.rados.0.smithi110.stdout:1278: writing smithi11075562-30 from 1843200 to 2613248 tid 1 2024-09-14T03:41:54.744 INFO:tasks.rados.rados.0.smithi110.stdout:1278: writing smithi11075562-30 from 2613248 to 3325952 tid 2 2024-09-14T03:41:54.748 INFO:tasks.rados.rados.0.smithi110.stdout:1278: writing smithi11075562-30 from 3325952 to 3817472 tid 3 2024-09-14T03:41:54.748 INFO:tasks.rados.rados.0.smithi110.stdout:1272: done (5 left) 2024-09-14T03:41:54.748 INFO:tasks.rados.rados.0.smithi110.stdout:1279: rollback oid 7 current snap is 132 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 7 to 104 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:1274: finishing rollback tid 1 to smithi11075562-36 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 859 (ObjNum 49 snap 0 seq_num 1) dirty dne 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:1271: finishing copy_from racing read to smithi11075562-35 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:1271: finishing copy_from to smithi11075562-35 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 477 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:1271: done (5 left) 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:1274: done (4 left) 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:1280: read oid 14 snap -1 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:1280: expect (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 132 2024-09-14T03:41:54.749 INFO:tasks.rados.rados.0.smithi110.stdout:1277: finishing write tid 1 to smithi11075562-21 2024-09-14T03:41:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1277: finishing write tid 2 to smithi11075562-21 2024-09-14T03:41:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1277: finishing write tid 3 to smithi11075562-21 2024-09-14T03:41:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1277: finishing write tid 4 to smithi11075562-21 2024-09-14T03:41:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1277: finishing write tid 5 to smithi11075562-21 2024-09-14T03:41:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1281: seq_num 405 ranges {2392064=786432,3178496=589824,3768320=614400} 2024-09-14T03:41:54.765 INFO:tasks.rados.rados.0.smithi110.stdout:1281: writing smithi11075562-50 from 2392064 to 3178496 tid 1 2024-09-14T03:41:54.769 INFO:tasks.rados.rados.0.smithi110.stdout:1281: writing smithi11075562-50 from 3178496 to 3768320 tid 2 2024-09-14T03:41:54.773 INFO:tasks.rados.rados.0.smithi110.stdout:1281: writing smithi11075562-50 from 3768320 to 4382720 tid 3 2024-09-14T03:41:54.773 INFO:tasks.rados.rados.0.smithi110.stdout:1282: snap_create 2024-09-14T03:41:54.774 INFO:tasks.rados.rados.0.smithi110.stdout:1277: finishing write tid 6 to smithi11075562-21 2024-09-14T03:41:54.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 716 (ObjNum 403 snap 132 seq_num 403) dirty exists 2024-09-14T03:41:54.774 INFO:tasks.rados.rados.0.smithi110.stdout:1277: left oid 21 (ObjNum 403 snap 132 seq_num 403) 2024-09-14T03:41:54.774 INFO:tasks.rados.rados.0.smithi110.stdout:1276: finishing copy_from racing read to smithi11075562-1 2024-09-14T03:41:54.774 INFO:tasks.rados.rados.0.smithi110.stdout:1276: finishing copy_from to smithi11075562-1 2024-09-14T03:41:54.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 494 (ObjNum 382 snap 125 seq_num 382) dirty exists 2024-09-14T03:41:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:1278: finishing write tid 1 to smithi11075562-30 2024-09-14T03:41:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:1278: finishing write tid 2 to smithi11075562-30 2024-09-14T03:41:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:1279: finishing rollback tid 1 to smithi11075562-7 2024-09-14T03:41:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 478 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-09-14T03:41:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:1278: finishing write tid 3 to smithi11075562-30 2024-09-14T03:41:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:1278: finishing write tid 4 to smithi11075562-30 2024-09-14T03:41:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:1278: finishing write tid 5 to smithi11075562-30 2024-09-14T03:41:54.776 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 737 (ObjNum 404 snap 132 seq_num 404) dirty exists 2024-09-14T03:41:54.776 INFO:tasks.rados.rados.0.smithi110.stdout:1278: left oid 30 (ObjNum 404 snap 132 seq_num 404) 2024-09-14T03:41:54.776 INFO:tasks.rados.rados.0.smithi110.stdout:1280: expect (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:41:54.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:54 smithi110 ceph-mon[88262]: osdmap e318: 8 total, 7 up, 8 in 2024-09-14T03:41:54.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:54 smithi110 ceph-mon[88262]: pgmap v103: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 12 op/s; 135/819 objects degraded (16.484%) 2024-09-14T03:41:54.946 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:54 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[93555]: 2024-09-14T03:41:54.636+0000 7f09bb2c3640 -1 Falling back to public interface 2024-09-14T03:41:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:54 smithi110 ceph-mon[86423]: osdmap e318: 8 total, 7 up, 8 in 2024-09-14T03:41:54.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:54 smithi110 ceph-mon[86423]: pgmap v103: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 12 op/s; 135/819 objects degraded (16.484%) 2024-09-14T03:41:54.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:54 smithi195 ceph-mon[82388]: osdmap e318: 8 total, 7 up, 8 in 2024-09-14T03:41:54.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:54 smithi195 ceph-mon[82388]: pgmap v103: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 12 op/s; 135/819 objects degraded (16.484%) 2024-09-14T03:41:55.053 INFO:tasks.rados.rados.0.smithi110.stdout:1281: finishing write tid 1 to smithi11075562-50 2024-09-14T03:41:55.053 INFO:tasks.rados.rados.0.smithi110.stdout:1281: finishing write tid 2 to smithi11075562-50 2024-09-14T03:41:55.053 INFO:tasks.rados.rados.0.smithi110.stdout:1281: finishing write tid 3 to smithi11075562-50 2024-09-14T03:41:55.053 INFO:tasks.rados.rados.0.smithi110.stdout:1281: finishing write tid 4 to smithi11075562-50 2024-09-14T03:41:55.053 INFO:tasks.rados.rados.0.smithi110.stdout:1281: finishing write tid 5 to smithi11075562-50 2024-09-14T03:41:55.053 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 114 (ObjNum 405 snap 132 seq_num 405) dirty exists 2024-09-14T03:41:55.053 INFO:tasks.rados.rados.0.smithi110.stdout:1281: left oid 50 (ObjNum 405 snap 132 seq_num 405) 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1276: done (6 left) 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1277: done (5 left) 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1278: done (4 left) 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1279: done (3 left) 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1280: done (2 left) 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1281: done (1 left) 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1282: done (0 left) 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1283: copy_from oid 26 from oid 22 current snap is 133 2024-09-14T03:41:55.708 INFO:tasks.rados.rados.0.smithi110.stdout:1284: snap_create 2024-09-14T03:41:55.714 INFO:tasks.rados.rados.0.smithi110.stdout:1283: finishing copy_from racing read to smithi11075562-26 2024-09-14T03:41:55.714 INFO:tasks.rados.rados.0.smithi110.stdout:1283: finishing copy_from to smithi11075562-26 2024-09-14T03:41:55.715 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 477 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:41:55.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:55 smithi195 ceph-mon[82388]: Health check failed: Degraded data redundancy: 135/819 objects degraded (16.484%), 16 pgs degraded (PG_DEGRADED) 2024-09-14T03:41:55.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:55 smithi195 ceph-mon[82388]: osdmap e319: 8 total, 7 up, 8 in 2024-09-14T03:41:55.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:55.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:56.067 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:55 smithi110 ceph-mon[88262]: Health check failed: Degraded data redundancy: 135/819 objects degraded (16.484%), 16 pgs degraded (PG_DEGRADED) 2024-09-14T03:41:56.068 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:55 smithi110 ceph-mon[88262]: osdmap e319: 8 total, 7 up, 8 in 2024-09-14T03:41:56.068 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:55 smithi110 ceph-mon[86423]: Health check failed: Degraded data redundancy: 135/819 objects degraded (16.484%), 16 pgs degraded (PG_DEGRADED) 2024-09-14T03:41:56.068 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:55 smithi110 ceph-mon[86423]: osdmap e319: 8 total, 7 up, 8 in 2024-09-14T03:41:56.713 INFO:tasks.rados.rados.0.smithi110.stdout:1283: done (1 left) 2024-09-14T03:41:56.713 INFO:tasks.rados.rados.0.smithi110.stdout:1284: done (0 left) 2024-09-14T03:41:56.713 INFO:tasks.rados.rados.0.smithi110.stdout:1285: read oid 49 snap -1 2024-09-14T03:41:56.713 INFO:tasks.rados.rados.0.smithi110.stdout:1285: expect deleted 2024-09-14T03:41:56.713 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 134 2024-09-14T03:41:56.713 INFO:tasks.rados.rados.0.smithi110.stdout:1286: seq_num 406 ranges {1507328=802816,2310144=475136,2785280=499712,3284992=229376} 2024-09-14T03:41:56.725 INFO:tasks.rados.rados.0.smithi110.stdout:1286: writing smithi11075562-45 from 1507328 to 2310144 tid 1 2024-09-14T03:41:56.728 INFO:tasks.rados.rados.0.smithi110.stdout:1286: writing smithi11075562-45 from 2310144 to 2785280 tid 2 2024-09-14T03:41:56.730 INFO:tasks.rados.rados.0.smithi110.stdout:1286: writing smithi11075562-45 from 2785280 to 3284992 tid 3 2024-09-14T03:41:56.732 INFO:tasks.rados.rados.0.smithi110.stdout:1286: writing smithi11075562-45 from 3284992 to 3514368 tid 4 2024-09-14T03:41:56.732 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 134 2024-09-14T03:41:56.732 INFO:tasks.rados.rados.0.smithi110.stdout:1287: seq_num 407 ranges {1859584=425984,2285568=761856,3047424=704512,3751936=139264} 2024-09-14T03:41:56.744 INFO:tasks.rados.rados.0.smithi110.stdout:1287: writing smithi11075562-18 from 1859584 to 2285568 tid 1 2024-09-14T03:41:56.749 INFO:tasks.rados.rados.0.smithi110.stdout:1287: writing smithi11075562-18 from 2285568 to 3047424 tid 2 2024-09-14T03:41:56.753 INFO:tasks.rados.rados.0.smithi110.stdout:1287: writing smithi11075562-18 from 3047424 to 3751936 tid 3 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1287: writing smithi11075562-18 from 3751936 to 3891200 tid 4 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1288: rollback oid 27 current snap is 134 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 106 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1286: finishing write tid 1 to smithi11075562-45 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1286: finishing write tid 2 to smithi11075562-45 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1286: finishing write tid 3 to smithi11075562-45 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1286: finishing write tid 4 to smithi11075562-45 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1286: finishing write tid 5 to smithi11075562-45 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1286: finishing write tid 6 to smithi11075562-45 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 865 (ObjNum 406 snap 134 seq_num 406) dirty exists 2024-09-14T03:41:56.755 INFO:tasks.rados.rados.0.smithi110.stdout:1286: left oid 45 (ObjNum 406 snap 134 seq_num 406) 2024-09-14T03:41:56.756 INFO:tasks.rados.rados.0.smithi110.stdout:1287: finishing write tid 1 to smithi11075562-18 2024-09-14T03:41:56.756 INFO:tasks.rados.rados.0.smithi110.stdout:1287: finishing write tid 2 to smithi11075562-18 2024-09-14T03:41:56.756 INFO:tasks.rados.rados.0.smithi110.stdout:1285: done (3 left) 2024-09-14T03:41:56.756 INFO:tasks.rados.rados.0.smithi110.stdout:1286: done (2 left) 2024-09-14T03:41:56.756 INFO:tasks.rados.rados.0.smithi110.stdout:1289: read oid 46 snap -1 2024-09-14T03:41:56.756 INFO:tasks.rados.rados.0.smithi110.stdout:1289: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:56.756 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 134 2024-09-14T03:41:56.756 INFO:tasks.rados.rados.0.smithi110.stdout:1290: seq_num 408 ranges {4120576=442368,4562944=647168,5210112=507904,5718016=450560} 2024-09-14T03:41:56.780 INFO:tasks.rados.rados.0.smithi110.stdout:1290: writing smithi11075562-44 from 4120576 to 4562944 tid 1 2024-09-14T03:41:56.784 INFO:tasks.rados.rados.0.smithi110.stdout:1290: writing smithi11075562-44 from 4562944 to 5210112 tid 2 2024-09-14T03:41:56.787 INFO:tasks.rados.rados.0.smithi110.stdout:1290: writing smithi11075562-44 from 5210112 to 5718016 tid 3 2024-09-14T03:41:56.790 INFO:tasks.rados.rados.0.smithi110.stdout:1290: writing smithi11075562-44 from 5718016 to 6168576 tid 4 2024-09-14T03:41:56.790 INFO:tasks.rados.rados.0.smithi110.stdout:1291: delete oid 8 current snap is 134 2024-09-14T03:41:56.790 INFO:tasks.rados.rados.0.smithi110.stdout:1288: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:41:56.790 INFO:tasks.rados.rados.0.smithi110.stdout:1288: finishing rollback tid 2 to smithi11075562-27 2024-09-14T03:41:56.790 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 130 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:41:56.790 INFO:tasks.rados.rados.0.smithi110.stdout:1287: finishing write tid 3 to smithi11075562-18 2024-09-14T03:41:56.791 INFO:tasks.rados.rados.0.smithi110.stdout:1287: finishing write tid 4 to smithi11075562-18 2024-09-14T03:41:56.791 INFO:tasks.rados.rados.0.smithi110.stdout:1287: finishing write tid 5 to smithi11075562-18 2024-09-14T03:41:56.791 INFO:tasks.rados.rados.0.smithi110.stdout:1287: finishing write tid 6 to smithi11075562-18 2024-09-14T03:41:56.791 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 640 (ObjNum 407 snap 134 seq_num 407) dirty exists 2024-09-14T03:41:56.791 INFO:tasks.rados.rados.0.smithi110.stdout:1287: left oid 18 (ObjNum 407 snap 134 seq_num 407) 2024-09-14T03:41:56.791 INFO:tasks.rados.rados.0.smithi110.stdout:1289: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:41:56.989 INFO:tasks.rados.rados.0.smithi110.stdout:1290: finishing write tid 1 to smithi11075562-44 2024-09-14T03:41:56.989 INFO:tasks.rados.rados.0.smithi110.stdout:1290: finishing write tid 2 to smithi11075562-44 2024-09-14T03:41:56.989 INFO:tasks.rados.rados.0.smithi110.stdout:1287: done (4 left) 2024-09-14T03:41:56.989 INFO:tasks.rados.rados.0.smithi110.stdout:1288: done (3 left) 2024-09-14T03:41:56.989 INFO:tasks.rados.rados.0.smithi110.stdout:1289: done (2 left) 2024-09-14T03:41:56.989 INFO:tasks.rados.rados.0.smithi110.stdout:1291: done (1 left) 2024-09-14T03:41:56.989 INFO:tasks.rados.rados.0.smithi110.stdout:1292: read oid 9 snap -1 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:1292: expect deleted 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 134 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:1290: finishing write tid 3 to smithi11075562-44 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:1290: finishing write tid 4 to smithi11075562-44 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:1290: finishing write tid 5 to smithi11075562-44 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:1290: finishing write tid 6 to smithi11075562-44 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 386 (ObjNum 408 snap 134 seq_num 408) dirty exists 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:1290: left oid 44 (ObjNum 408 snap 134 seq_num 408) 2024-09-14T03:41:56.990 INFO:tasks.rados.rados.0.smithi110.stdout:1293: seq_num 410 ranges {4382720=475136,4857856=417792,5275648=475136,5750784=425984,6176768=286720} 2024-09-14T03:41:57.016 INFO:tasks.rados.rados.0.smithi110.stdout:1293: writing smithi11075562-50 from 4382720 to 4857856 tid 1 2024-09-14T03:41:57.019 INFO:tasks.rados.rados.0.smithi110.stdout:1293: writing smithi11075562-50 from 4857856 to 5275648 tid 2 2024-09-14T03:41:57.023 INFO:tasks.rados.rados.0.smithi110.stdout:1293: writing smithi11075562-50 from 5275648 to 5750784 tid 3 2024-09-14T03:41:57.026 INFO:tasks.rados.rados.0.smithi110.stdout:1293: writing smithi11075562-50 from 5750784 to 6176768 tid 4 2024-09-14T03:41:57.028 INFO:tasks.rados.rados.0.smithi110.stdout:1293: writing smithi11075562-50 from 6176768 to 6463488 tid 5 2024-09-14T03:41:57.028 INFO:tasks.rados.rados.0.smithi110.stdout:1290: done (2 left) 2024-09-14T03:41:57.029 INFO:tasks.rados.rados.0.smithi110.stdout:1294: rmattr oid 20 current snap is 134 2024-09-14T03:41:57.029 INFO:tasks.rados.rados.0.smithi110.stdout:1293: finishing write tid 1 to smithi11075562-50 2024-09-14T03:41:57.029 INFO:tasks.rados.rados.0.smithi110.stdout:1293: finishing write tid 2 to smithi11075562-50 2024-09-14T03:41:57.029 INFO:tasks.rados.rados.0.smithi110.stdout:1293: finishing write tid 3 to smithi11075562-50 2024-09-14T03:41:57.029 INFO:tasks.rados.rados.0.smithi110.stdout:1292: done (2 left) 2024-09-14T03:41:57.029 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 134 2024-09-14T03:41:57.029 INFO:tasks.rados.rados.0.smithi110.stdout:1295: seq_num 411 ranges {1933312=491520,2424832=753664,3178496=688128,3866624=163840} 2024-09-14T03:41:57.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:56 smithi110 ceph-mon[88262]: osdmap e320: 8 total, 7 up, 8 in 2024-09-14T03:41:57.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:56 smithi110 ceph-mon[88262]: pgmap v106: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 12 op/s; 135/819 objects degraded (16.484%) 2024-09-14T03:41:57.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:56 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:56 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:56 smithi110 ceph-mon[86423]: osdmap e320: 8 total, 7 up, 8 in 2024-09-14T03:41:57.042 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:56 smithi110 ceph-mon[86423]: pgmap v106: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 12 op/s; 135/819 objects degraded (16.484%) 2024-09-14T03:41:57.042 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:56 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.042 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:56 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.043 INFO:tasks.rados.rados.0.smithi110.stdout:1295: writing smithi11075562-40 from 1933312 to 2424832 tid 1 2024-09-14T03:41:57.048 INFO:tasks.rados.rados.0.smithi110.stdout:1295: writing smithi11075562-40 from 2424832 to 3178496 tid 2 2024-09-14T03:41:57.053 INFO:tasks.rados.rados.0.smithi110.stdout:1295: writing smithi11075562-40 from 3178496 to 3866624 tid 3 2024-09-14T03:41:57.055 INFO:tasks.rados.rados.0.smithi110.stdout:1295: writing smithi11075562-40 from 3866624 to 4030464 tid 4 2024-09-14T03:41:57.055 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 134 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:1293: finishing write tid 4 to smithi11075562-50 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:1293: finishing write tid 5 to smithi11075562-50 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:1293: finishing write tid 6 to smithi11075562-50 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 509 (ObjNum 292 snap 90 seq_num 292) dirty exists 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:1293: finishing write tid 7 to smithi11075562-50 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 121 (ObjNum 410 snap 134 seq_num 410) dirty exists 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:1293: left oid 50 (ObjNum 410 snap 134 seq_num 410) 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:1295: finishing write tid 1 to smithi11075562-40 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:1295: finishing write tid 2 to smithi11075562-40 2024-09-14T03:41:57.056 INFO:tasks.rados.rados.0.smithi110.stdout:1296: seq_num 412 ranges {4030464=507904,4538368=630784,5169152=491520,5660672=483328} 2024-09-14T03:41:57.081 INFO:tasks.rados.rados.0.smithi110.stdout:1296: writing smithi11075562-11 from 4030464 to 4538368 tid 1 2024-09-14T03:41:57.086 INFO:tasks.rados.rados.0.smithi110.stdout:1296: writing smithi11075562-11 from 4538368 to 5169152 tid 2 2024-09-14T03:41:57.089 INFO:tasks.rados.rados.0.smithi110.stdout:1296: writing smithi11075562-11 from 5169152 to 5660672 tid 3 2024-09-14T03:41:57.091 INFO:tasks.rados.rados.0.smithi110.stdout:1296: writing smithi11075562-11 from 5660672 to 6144000 tid 4 2024-09-14T03:41:57.091 INFO:tasks.rados.rados.0.smithi110.stdout:1293: done (3 left) 2024-09-14T03:41:57.092 INFO:tasks.rados.rados.0.smithi110.stdout:1294: done (2 left) 2024-09-14T03:41:57.092 INFO:tasks.rados.rados.0.smithi110.stdout:1297: snap_remove snap 117 2024-09-14T03:41:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:56 smithi195 ceph-mon[82388]: osdmap e320: 8 total, 7 up, 8 in 2024-09-14T03:41:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:56 smithi195 ceph-mon[82388]: pgmap v106: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 12 op/s; 135/819 objects degraded (16.484%) 2024-09-14T03:41:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:56 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:56 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:41:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:41:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:41:57.866 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[86423]: osdmap e321: 8 total, 7 up, 8 in 2024-09-14T03:41:57.866 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.867 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:57.867 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.867 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.867 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[88262]: osdmap e321: 8 total, 7 up, 8 in 2024-09-14T03:41:57.867 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.867 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:57.867 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:57.867 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:1297: done (2 left) 2024-09-14T03:41:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:1298: snap_remove snap 105 2024-09-14T03:41:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:57 smithi195 ceph-mon[82388]: osdmap e321: 8 total, 7 up, 8 in 2024-09-14T03:41:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:41:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:41:58.696 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[93555]: 2024-09-14T03:41:58.372+0000 7f09bb2c3640 -1 osd.0 311 log_to_monitors true 2024-09-14T03:41:58.983 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:58 smithi110 ceph-mon[88262]: pgmap v108: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 6.9 MiB/s wr, 20 op/s; 140/864 objects degraded (16.204%) 2024-09-14T03:41:58.983 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:58 smithi110 ceph-mon[88262]: osdmap e322: 8 total, 7 up, 8 in 2024-09-14T03:41:58.983 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:41:58 smithi110 ceph-mon[88262]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:41:58.983 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:58 smithi110 ceph-mon[86423]: pgmap v108: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 6.9 MiB/s wr, 20 op/s; 140/864 objects degraded (16.204%) 2024-09-14T03:41:58.983 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:58 smithi110 ceph-mon[86423]: osdmap e322: 8 total, 7 up, 8 in 2024-09-14T03:41:58.984 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:41:58 smithi110 ceph-mon[86423]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:41:59.033 INFO:tasks.rados.rados.0.smithi110.stdout:1298: done (2 left) 2024-09-14T03:41:59.033 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 134 2024-09-14T03:41:59.033 INFO:tasks.rados.rados.0.smithi110.stdout:1299: seq_num 413 ranges {1908736=524288,2433024=516096,2949120=696320,3645440=393216} 2024-09-14T03:41:59.047 INFO:tasks.rados.rados.0.smithi110.stdout:1299: writing smithi11075562-43 from 1908736 to 2433024 tid 1 2024-09-14T03:41:59.050 INFO:tasks.rados.rados.0.smithi110.stdout:1299: writing smithi11075562-43 from 2433024 to 2949120 tid 2 2024-09-14T03:41:59.055 INFO:tasks.rados.rados.0.smithi110.stdout:1299: writing smithi11075562-43 from 2949120 to 3645440 tid 3 2024-09-14T03:41:59.058 INFO:tasks.rados.rados.0.smithi110.stdout:1299: writing smithi11075562-43 from 3645440 to 4038656 tid 4 2024-09-14T03:41:59.058 INFO:tasks.rados.rados.0.smithi110.stdout:1300: read oid 28 snap 127 2024-09-14T03:41:59.058 INFO:tasks.rados.rados.0.smithi110.stdout:1300: expect (ObjNum 386 snap 127 seq_num 386) 2024-09-14T03:41:59.058 INFO:tasks.rados.rados.0.smithi110.stdout:1301: rmattr oid 15 current snap is 134 2024-09-14T03:41:59.058 INFO:tasks.rados.rados.0.smithi110.stdout:1301: done (4 left) 2024-09-14T03:41:59.058 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 134 2024-09-14T03:41:59.058 INFO:tasks.rados.rados.0.smithi110.stdout:1295: finishing write tid 3 to smithi11075562-40 2024-09-14T03:41:59.059 INFO:tasks.rados.rados.0.smithi110.stdout:1295: finishing write tid 4 to smithi11075562-40 2024-09-14T03:41:59.059 INFO:tasks.rados.rados.0.smithi110.stdout:1302: seq_num 414 ranges {2383872=540672,2924544=802816,3727360=458752,4186112=344064} 2024-09-14T03:41:59.074 INFO:tasks.rados.rados.0.smithi110.stdout:1302: writing smithi11075562-2 from 2383872 to 2924544 tid 1 2024-09-14T03:41:59.079 INFO:tasks.rados.rados.0.smithi110.stdout:1302: writing smithi11075562-2 from 2924544 to 3727360 tid 2 2024-09-14T03:41:59.082 INFO:tasks.rados.rados.0.smithi110.stdout:1302: writing smithi11075562-2 from 3727360 to 4186112 tid 3 2024-09-14T03:41:59.085 INFO:tasks.rados.rados.0.smithi110.stdout:1302: writing smithi11075562-2 from 4186112 to 4530176 tid 4 2024-09-14T03:41:59.085 INFO:tasks.rados.rados.0.smithi110.stdout:1303: read oid 26 snap -1 2024-09-14T03:41:59.085 INFO:tasks.rados.rados.0.smithi110.stdout:1303: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:41:59.085 INFO:tasks.rados.rados.0.smithi110.stdout:1304: delete oid 25 current snap is 134 2024-09-14T03:41:59.085 INFO:tasks.rados.rados.0.smithi110.stdout:1295: finishing write tid 5 to smithi11075562-40 2024-09-14T03:41:59.085 INFO:tasks.rados.rados.0.smithi110.stdout:1295: finishing write tid 6 to smithi11075562-40 2024-09-14T03:41:59.085 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 646 (ObjNum 411 snap 134 seq_num 411) dirty exists 2024-09-14T03:41:59.085 INFO:tasks.rados.rados.0.smithi110.stdout:1295: left oid 40 (ObjNum 411 snap 134 seq_num 411) 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1296: finishing write tid 1 to smithi11075562-11 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1296: finishing write tid 2 to smithi11075562-11 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1296: finishing write tid 3 to smithi11075562-11 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1296: finishing write tid 4 to smithi11075562-11 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1296: finishing write tid 5 to smithi11075562-11 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1296: finishing write tid 6 to smithi11075562-11 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 627 (ObjNum 412 snap 134 seq_num 412) dirty exists 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1296: left oid 11 (ObjNum 412 snap 134 seq_num 412) 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1299: finishing write tid 1 to smithi11075562-43 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1299: finishing write tid 2 to smithi11075562-43 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1299: finishing write tid 3 to smithi11075562-43 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1299: finishing write tid 4 to smithi11075562-43 2024-09-14T03:41:59.086 INFO:tasks.rados.rados.0.smithi110.stdout:1299: finishing write tid 5 to smithi11075562-43 2024-09-14T03:41:59.087 INFO:tasks.rados.rados.0.smithi110.stdout:1299: finishing write tid 6 to smithi11075562-43 2024-09-14T03:41:59.087 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 722 (ObjNum 413 snap 134 seq_num 413) dirty exists 2024-09-14T03:41:59.087 INFO:tasks.rados.rados.0.smithi110.stdout:1299: left oid 43 (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:41:59.087 INFO:tasks.rados.rados.0.smithi110.stdout:1300: expect (ObjNum 386 snap 127 seq_num 386) 2024-09-14T03:41:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:58 smithi195 ceph-mon[82388]: pgmap v108: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 6.9 MiB/s wr, 20 op/s; 140/864 objects degraded (16.204%) 2024-09-14T03:41:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:58 smithi195 ceph-mon[82388]: osdmap e322: 8 total, 7 up, 8 in 2024-09-14T03:41:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:41:58 smithi195 ceph-mon[82388]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:41:59.364 INFO:tasks.rados.rados.0.smithi110.stdout:1302: finishing write tid 1 to smithi11075562-2 2024-09-14T03:41:59.364 INFO:tasks.rados.rados.0.smithi110.stdout:1302: finishing write tid 2 to smithi11075562-2 2024-09-14T03:41:59.364 INFO:tasks.rados.rados.0.smithi110.stdout:1295: done (6 left) 2024-09-14T03:41:59.364 INFO:tasks.rados.rados.0.smithi110.stdout:1296: done (5 left) 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1299: done (4 left) 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1300: done (3 left) 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1304: done (2 left) 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1305: rollback oid 47 current snap is 134 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 47 to 116 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1302: finishing write tid 3 to smithi11075562-2 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1302: finishing write tid 4 to smithi11075562-2 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1302: finishing write tid 5 to smithi11075562-2 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1302: finishing write tid 6 to smithi11075562-2 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 668 (ObjNum 414 snap 134 seq_num 414) dirty exists 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1302: left oid 2 (ObjNum 414 snap 134 seq_num 414) 2024-09-14T03:41:59.365 INFO:tasks.rados.rados.0.smithi110.stdout:1303: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:41:59.379 INFO:tasks.rados.rados.0.smithi110.stdout:1305: finishing rollback tid 0 to smithi11075562-47 2024-09-14T03:41:59.379 INFO:tasks.rados.rados.0.smithi110.stdout:1305: finishing rollback tid 1 to smithi11075562-47 2024-09-14T03:41:59.380 INFO:tasks.rados.rados.0.smithi110.stdout:1302: done (2 left) 2024-09-14T03:41:59.380 INFO:tasks.rados.rados.0.smithi110.stdout:1303: done (1 left) 2024-09-14T03:41:59.380 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 134 2024-09-14T03:41:59.380 INFO:tasks.rados.rados.0.smithi110.stdout:1305: finishing rollback tid 2 to smithi11075562-47 2024-09-14T03:41:59.380 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 879 (ObjNum 305 snap 98 seq_num 305) dirty exists 2024-09-14T03:41:59.380 INFO:tasks.rados.rados.0.smithi110.stdout:1306: seq_num 416 ranges {4530176=573440,5103616=573440,5677056=475136,6152192=557056} 2024-09-14T03:41:59.408 INFO:tasks.rados.rados.0.smithi110.stdout:1306: writing smithi11075562-2 from 4530176 to 5103616 tid 1 2024-09-14T03:41:59.411 INFO:tasks.rados.rados.0.smithi110.stdout:1306: writing smithi11075562-2 from 5103616 to 5677056 tid 2 2024-09-14T03:41:59.414 INFO:tasks.rados.rados.0.smithi110.stdout:1306: writing smithi11075562-2 from 5677056 to 6152192 tid 3 2024-09-14T03:41:59.419 INFO:tasks.rados.rados.0.smithi110.stdout:1306: writing smithi11075562-2 from 6152192 to 6709248 tid 4 2024-09-14T03:41:59.419 INFO:tasks.rados.rados.0.smithi110.stdout:1305: done (1 left) 2024-09-14T03:41:59.419 INFO:tasks.rados.rados.0.smithi110.stdout:1307: read oid 42 snap -1 2024-09-14T03:41:59.419 INFO:tasks.rados.rados.0.smithi110.stdout:1307: expect deleted 2024-09-14T03:41:59.420 INFO:tasks.rados.rados.0.smithi110.stdout:1308: read oid 1 snap 121 2024-09-14T03:41:59.420 INFO:tasks.rados.rados.0.smithi110.stdout:1308: expect (ObjNum 335 snap 106 seq_num 335) 2024-09-14T03:41:59.420 INFO:tasks.rados.rados.0.smithi110.stdout:1309: snap_remove snap 116 2024-09-14T03:41:59.446 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:41:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[93555]: 2024-09-14T03:41:59.054+0000 7f09b306d640 -1 osd.0 311 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:42:00.035 INFO:tasks.rados.rados.0.smithi110.stdout:1309: done (3 left) 2024-09-14T03:42:00.035 INFO:tasks.rados.rados.0.smithi110.stdout:1310: read oid 38 snap -1 2024-09-14T03:42:00.035 INFO:tasks.rados.rados.0.smithi110.stdout:1310: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:42:00.035 INFO:tasks.rados.rados.0.smithi110.stdout:1311: snap_create 2024-09-14T03:42:00.035 INFO:tasks.rados.rados.0.smithi110.stdout:1306: finishing write tid 1 to smithi11075562-2 2024-09-14T03:42:00.036 INFO:tasks.rados.rados.0.smithi110.stdout:1306: finishing write tid 2 to smithi11075562-2 2024-09-14T03:42:00.036 INFO:tasks.rados.rados.0.smithi110.stdout:1306: finishing write tid 3 to smithi11075562-2 2024-09-14T03:42:00.036 INFO:tasks.rados.rados.0.smithi110.stdout:1306: finishing write tid 4 to smithi11075562-2 2024-09-14T03:42:00.036 INFO:tasks.rados.rados.0.smithi110.stdout:1306: finishing write tid 5 to smithi11075562-2 2024-09-14T03:42:00.036 INFO:tasks.rados.rados.0.smithi110.stdout:1306: finishing write tid 6 to smithi11075562-2 2024-09-14T03:42:00.036 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 673 (ObjNum 416 snap 134 seq_num 416) dirty exists 2024-09-14T03:42:00.036 INFO:tasks.rados.rados.0.smithi110.stdout:1306: left oid 2 (ObjNum 416 snap 134 seq_num 416) 2024-09-14T03:42:00.036 INFO:tasks.rados.rados.0.smithi110.stdout:1308: expect (ObjNum 335 snap 106 seq_num 335) 2024-09-14T03:42:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-mon[86423]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:42:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-mon[86423]: osdmap e323: 8 total, 7 up, 8 in 2024-09-14T03:42:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-mon[86423]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-mon[86423]: pgmap v111: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 6.9 MiB/s wr, 20 op/s; 140/864 objects degraded (16.204%) 2024-09-14T03:42:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-mon[88262]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:42:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-mon[88262]: osdmap e323: 8 total, 7 up, 8 in 2024-09-14T03:42:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-mon[88262]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-mon[88262]: pgmap v111: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 6.9 MiB/s wr, 20 op/s; 140/864 objects degraded (16.204%) 2024-09-14T03:42:00.197 INFO:tasks.rados.rados.0.smithi110.stdout:1310: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:42:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:00 smithi195 ceph-mon[82388]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:42:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:00 smithi195 ceph-mon[82388]: osdmap e323: 8 total, 7 up, 8 in 2024-09-14T03:42:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:00 smithi195 ceph-mon[82388]: from='osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:00 smithi195 ceph-mon[82388]: pgmap v111: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 6.9 MiB/s wr, 20 op/s; 140/864 objects degraded (16.204%) 2024-09-14T03:42:00.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:00.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:42:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:42:00.368+0000 7f940e88b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:42:01.042 INFO:tasks.rados.rados.0.smithi110.stdout:1306: done (4 left) 2024-09-14T03:42:01.042 INFO:tasks.rados.rados.0.smithi110.stdout:1307: done (3 left) 2024-09-14T03:42:01.043 INFO:tasks.rados.rados.0.smithi110.stdout:1308: done (2 left) 2024-09-14T03:42:01.043 INFO:tasks.rados.rados.0.smithi110.stdout:1310: done (1 left) 2024-09-14T03:42:01.043 INFO:tasks.rados.rados.0.smithi110.stdout:1311: done (0 left) 2024-09-14T03:42:01.043 INFO:tasks.rados.rados.0.smithi110.stdout:1312: snap_create 2024-09-14T03:42:01.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:42:01.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393] boot 2024-09-14T03:42:01.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:42:01.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:42:01.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:42:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:42:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:42:01.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[86423]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:42:01.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:42:01.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393] boot 2024-09-14T03:42:01.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:42:01.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:42:01.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:01.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:42:01.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:01.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:42:01.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:42:01.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-mon[88262]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: osd.0 [v2:172.21.15.110:6802/4018695393,v1:172.21.15.110:6803/4018695393] boot 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:01.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:01.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:42:01.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:42:01.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:01 smithi195 ceph-mon[82388]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:42:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:42:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:42:01] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-14T03:42:02.045 INFO:tasks.rados.rados.0.smithi110.stdout:1312: done (0 left) 2024-09-14T03:42:02.046 INFO:tasks.rados.rados.0.smithi110.stdout:1313: read oid 17 snap 103 2024-09-14T03:42:02.046 INFO:tasks.rados.rados.0.smithi110.stdout:1313: expect (ObjNum 314 snap 100 seq_num 314) 2024-09-14T03:42:02.046 INFO:tasks.rados.rados.0.smithi110.stdout:1314: delete oid 15 current snap is 136 2024-09-14T03:42:02.049 INFO:tasks.rados.rados.0.smithi110.stdout:1314: done (1 left) 2024-09-14T03:42:02.049 INFO:tasks.rados.rados.0.smithi110.stdout:1315: read oid 2 snap 90 2024-09-14T03:42:02.049 INFO:tasks.rados.rados.0.smithi110.stdout:1315: expect (ObjNum 277 snap 82 seq_num 277) 2024-09-14T03:42:02.049 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 136 2024-09-14T03:42:02.049 INFO:tasks.rados.rados.0.smithi110.stdout:1316: seq_num 418 ranges {0=606208,606208=737280,1343488=434176,1777664=434176} 2024-09-14T03:42:02.053 INFO:tasks.rados.rados.0.smithi110.stdout:1316: writing smithi11075562-8 from 0 to 606208 tid 1 2024-09-14T03:42:02.058 INFO:tasks.rados.rados.0.smithi110.stdout:1316: writing smithi11075562-8 from 606208 to 1343488 tid 2 2024-09-14T03:42:02.061 INFO:tasks.rados.rados.0.smithi110.stdout:1316: writing smithi11075562-8 from 1343488 to 1777664 tid 3 2024-09-14T03:42:02.064 INFO:tasks.rados.rados.0.smithi110.stdout:1316: writing smithi11075562-8 from 1777664 to 2211840 tid 4 2024-09-14T03:42:02.064 INFO:tasks.rados.rados.0.smithi110.stdout:1317: rollback oid 35 current snap is 136 2024-09-14T03:42:02.064 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 107 2024-09-14T03:42:02.064 INFO:tasks.rados.rados.0.smithi110.stdout:1313: expect (ObjNum 314 snap 100 seq_num 314) 2024-09-14T03:42:02.077 INFO:tasks.rados.rados.0.smithi110.stdout:1316: finishing write tid 1 to smithi11075562-8 2024-09-14T03:42:02.077 INFO:tasks.rados.rados.0.smithi110.stdout:1315: expect (ObjNum 277 snap 82 seq_num 277) 2024-09-14T03:42:02.135 INFO:tasks.rados.rados.0.smithi110.stdout:1316: finishing write tid 2 to smithi11075562-8 2024-09-14T03:42:02.135 INFO:tasks.rados.rados.0.smithi110.stdout:1316: finishing write tid 3 to smithi11075562-8 2024-09-14T03:42:02.135 INFO:tasks.rados.rados.0.smithi110.stdout:1317: finishing rollback tid 0 to smithi11075562-35 2024-09-14T03:42:02.135 INFO:tasks.rados.rados.0.smithi110.stdout:1316: finishing write tid 4 to smithi11075562-8 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1317: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 787 (ObjNum 289 snap 0 seq_num 1494916441) dirty dne 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1316: finishing write tid 5 to smithi11075562-8 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1316: finishing write tid 6 to smithi11075562-8 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 763 (ObjNum 418 snap 136 seq_num 418) dirty exists 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1316: left oid 8 (ObjNum 418 snap 136 seq_num 418) 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1313: done (3 left) 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1315: done (2 left) 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1316: done (1 left) 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1317: done (0 left) 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1318: rmattr oid 35 current snap is 136 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1318: done (0 left) 2024-09-14T03:42:02.136 INFO:tasks.rados.rados.0.smithi110.stdout:1319: setattr oid 19 current snap is 136 2024-09-14T03:42:02.137 INFO:tasks.rados.rados.0.smithi110.stdout:1320: snap_remove snap 120 2024-09-14T03:42:02.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:02 smithi110 ceph-mon[86423]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:42:02.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:02 smithi110 ceph-mon[86423]: pgmap v114: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 140/864 objects degraded (16.204%) 2024-09-14T03:42:02.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:02 smithi110 ceph-mon[88262]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:42:02.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:02 smithi110 ceph-mon[88262]: pgmap v114: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 140/864 objects degraded (16.204%) 2024-09-14T03:42:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:02 smithi195 ceph-mon[82388]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:42:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:02 smithi195 ceph-mon[82388]: pgmap v114: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 360 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 140/864 objects degraded (16.204%) 2024-09-14T03:42:03.050 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 674 (ObjNum 355 snap 115 seq_num 355) dirty exists 2024-09-14T03:42:03.050 INFO:tasks.rados.rados.0.smithi110.stdout:1319: done (1 left) 2024-09-14T03:42:03.050 INFO:tasks.rados.rados.0.smithi110.stdout:1320: done (0 left) 2024-09-14T03:42:03.050 INFO:tasks.rados.rados.0.smithi110.stdout:1321: rmattr oid 30 current snap is 136 2024-09-14T03:42:03.050 INFO:tasks.rados.rados.0.smithi110.stdout:1321: done (0 left) 2024-09-14T03:42:03.051 INFO:tasks.rados.rados.0.smithi110.stdout:1322: snap_remove snap 104 2024-09-14T03:42:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:03 smithi110 ceph-mon[86423]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:42:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:03 smithi110 ceph-mon[88262]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:42:03.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:03 smithi195 ceph-mon[82388]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:42:04.054 INFO:tasks.rados.rados.0.smithi110.stdout:1322: done (0 left) 2024-09-14T03:42:04.055 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 136 2024-09-14T03:42:04.055 INFO:tasks.rados.rados.0.smithi110.stdout:1323: seq_num 419 ranges {6144000=622592,6766592=671744,7438336=647168,8085504=286720} 2024-09-14T03:42:04.092 INFO:tasks.rados.rados.0.smithi110.stdout:1323: writing smithi11075562-11 from 6144000 to 6766592 tid 1 2024-09-14T03:42:04.095 INFO:tasks.rados.rados.0.smithi110.stdout:1323: writing smithi11075562-11 from 6766592 to 7438336 tid 2 2024-09-14T03:42:04.099 INFO:tasks.rados.rados.0.smithi110.stdout:1323: writing smithi11075562-11 from 7438336 to 8085504 tid 3 2024-09-14T03:42:04.101 INFO:tasks.rados.rados.0.smithi110.stdout:1323: writing smithi11075562-11 from 8085504 to 8372224 tid 4 2024-09-14T03:42:04.101 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 136 2024-09-14T03:42:04.101 INFO:tasks.rados.rados.0.smithi110.stdout:1323: finishing write tid 1 to smithi11075562-11 2024-09-14T03:42:04.101 INFO:tasks.rados.rados.0.smithi110.stdout:1323: finishing write tid 2 to smithi11075562-11 2024-09-14T03:42:04.101 INFO:tasks.rados.rados.0.smithi110.stdout:1324: seq_num 420 ranges {4120576=638976,4759552=557056,5316608=450560,5767168=598016} 2024-09-14T03:42:04.126 INFO:tasks.rados.rados.0.smithi110.stdout:1324: writing smithi11075562-3 from 4120576 to 4759552 tid 1 2024-09-14T03:42:04.129 INFO:tasks.rados.rados.0.smithi110.stdout:1324: writing smithi11075562-3 from 4759552 to 5316608 tid 2 2024-09-14T03:42:04.132 INFO:tasks.rados.rados.0.smithi110.stdout:1324: writing smithi11075562-3 from 5316608 to 5767168 tid 3 2024-09-14T03:42:04.137 INFO:tasks.rados.rados.0.smithi110.stdout:1324: writing smithi11075562-3 from 5767168 to 6365184 tid 4 2024-09-14T03:42:04.140 INFO:tasks.rados.rados.0.smithi110.stdout:1325: rollback oid 13 current snap is 136 2024-09-14T03:42:04.140 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 106 2024-09-14T03:42:04.140 INFO:tasks.rados.rados.0.smithi110.stdout:1323: finishing write tid 3 to smithi11075562-11 2024-09-14T03:42:04.140 INFO:tasks.rados.rados.0.smithi110.stdout:1323: finishing write tid 4 to smithi11075562-11 2024-09-14T03:42:04.140 INFO:tasks.rados.rados.0.smithi110.stdout:1323: finishing write tid 5 to smithi11075562-11 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1323: finishing write tid 6 to smithi11075562-11 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 632 (ObjNum 419 snap 136 seq_num 419) dirty exists 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1323: left oid 11 (ObjNum 419 snap 136 seq_num 419) 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1324: finishing write tid 1 to smithi11075562-3 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1324: finishing write tid 2 to smithi11075562-3 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1324: finishing write tid 3 to smithi11075562-3 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1323: done (2 left) 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1326: rollback oid 25 current snap is 136 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 25 to 113 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1327: rmattr oid 46 current snap is 136 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1328: rmattr oid 18 current snap is 136 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1328: done (4 left) 2024-09-14T03:42:04.141 INFO:tasks.rados.rados.0.smithi110.stdout:1329: delete oid 12 current snap is 136 2024-09-14T03:42:04.142 INFO:tasks.rados.rados.0.smithi110.stdout:1324: finishing write tid 4 to smithi11075562-3 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:1324: finishing write tid 5 to smithi11075562-3 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 535 (ObjNum 313 snap 100 seq_num 313) dirty exists 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:1326: finishing rollback tid 1 to smithi11075562-25 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 871 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:1324: finishing write tid 6 to smithi11075562-3 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 678 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:1324: left oid 3 (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:1324: done (4 left) 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:1326: done (3 left) 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:1327: done (2 left) 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:1329: done (1 left) 2024-09-14T03:42:04.143 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 136 2024-09-14T03:42:04.144 INFO:tasks.rados.rados.0.smithi110.stdout:1330: seq_num 422 ranges {4120576=671744,4792320=729088,5521408=417792,5939200=458752} 2024-09-14T03:42:04.166 INFO:tasks.rados.rados.0.smithi110.stdout:1330: writing smithi11075562-46 from 4120576 to 4792320 tid 1 2024-09-14T03:42:04.170 INFO:tasks.rados.rados.0.smithi110.stdout:1330: writing smithi11075562-46 from 4792320 to 5521408 tid 2 2024-09-14T03:42:04.173 INFO:tasks.rados.rados.0.smithi110.stdout:1330: writing smithi11075562-46 from 5521408 to 5939200 tid 3 2024-09-14T03:42:04.175 INFO:tasks.rados.rados.0.smithi110.stdout:1330: writing smithi11075562-46 from 5939200 to 6397952 tid 4 2024-09-14T03:42:04.175 INFO:tasks.rados.rados.0.smithi110.stdout:1331: snap_remove snap 103 2024-09-14T03:42:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:04 smithi110 ceph-mon[86423]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:42:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:04 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 140/864 objects degraded (16.204%), 16 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:04 smithi110 ceph-mon[86423]: pgmap v117: 121 pgs: 10 active+undersized, 5 active+undersized+degraded, 106 active+clean; 357 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 769 KiB/s rd, 1.0 MiB/s wr, 10 op/s; 26/858 objects degraded (3.030%); 19 MiB/s, 5 objects/s recovering 2024-09-14T03:42:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:04 smithi110 ceph-mon[88262]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:42:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:04 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 140/864 objects degraded (16.204%), 16 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:04 smithi110 ceph-mon[88262]: pgmap v117: 121 pgs: 10 active+undersized, 5 active+undersized+degraded, 106 active+clean; 357 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 769 KiB/s rd, 1.0 MiB/s wr, 10 op/s; 26/858 objects degraded (3.030%); 19 MiB/s, 5 objects/s recovering 2024-09-14T03:42:04.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:04 smithi195 ceph-mon[82388]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:42:04.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:04 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 140/864 objects degraded (16.204%), 16 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:04.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:04 smithi195 ceph-mon[82388]: pgmap v117: 121 pgs: 10 active+undersized, 5 active+undersized+degraded, 106 active+clean; 357 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 769 KiB/s rd, 1.0 MiB/s wr, 10 op/s; 26/858 objects degraded (3.030%); 19 MiB/s, 5 objects/s recovering 2024-09-14T03:42:04.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:04.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:42:04.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:1331: done (2 left) 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:1332: read oid 39 snap -1 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:1332: expect deleted 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:1333: copy_from oid 14 from oid 43 current snap is 136 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:1325: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:1325: finishing rollback tid 2 to smithi11075562-13 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 803 (ObjNum 287 snap 86 seq_num 287) dirty exists 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:1330: finishing write tid 1 to smithi11075562-46 2024-09-14T03:42:05.060 INFO:tasks.rados.rados.0.smithi110.stdout:1330: finishing write tid 2 to smithi11075562-46 2024-09-14T03:42:05.061 INFO:tasks.rados.rados.0.smithi110.stdout:1325: done (3 left) 2024-09-14T03:42:05.061 INFO:tasks.rados.rados.0.smithi110.stdout:1334: delete oid 31 current snap is 136 2024-09-14T03:42:05.061 INFO:tasks.rados.rados.0.smithi110.stdout:1330: finishing write tid 3 to smithi11075562-46 2024-09-14T03:42:05.061 INFO:tasks.rados.rados.0.smithi110.stdout:1330: finishing write tid 4 to smithi11075562-46 2024-09-14T03:42:05.061 INFO:tasks.rados.rados.0.smithi110.stdout:1330: finishing write tid 5 to smithi11075562-46 2024-09-14T03:42:05.061 INFO:tasks.rados.rados.0.smithi110.stdout:1330: finishing write tid 6 to smithi11075562-46 2024-09-14T03:42:05.061 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 540 (ObjNum 422 snap 136 seq_num 422) dirty exists 2024-09-14T03:42:05.061 INFO:tasks.rados.rados.0.smithi110.stdout:1330: left oid 46 (ObjNum 422 snap 136 seq_num 422) 2024-09-14T03:42:05.072 INFO:tasks.rados.rados.0.smithi110.stdout:1330: done (3 left) 2024-09-14T03:42:05.072 INFO:tasks.rados.rados.0.smithi110.stdout:1332: done (2 left) 2024-09-14T03:42:05.072 INFO:tasks.rados.rados.0.smithi110.stdout:1334: done (1 left) 2024-09-14T03:42:05.072 INFO:tasks.rados.rados.0.smithi110.stdout:1335: copy_from oid 25 from oid 3 current snap is 136 2024-09-14T03:42:05.072 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 136 2024-09-14T03:42:05.072 INFO:tasks.rados.rados.0.smithi110.stdout:1336: seq_num 424 ranges {5726208=704512,6430720=491520,6922240=434176,7356416=679936} 2024-09-14T03:42:05.108 INFO:tasks.rados.rados.0.smithi110.stdout:1336: writing smithi11075562-1 from 5726208 to 6430720 tid 1 2024-09-14T03:42:05.111 INFO:tasks.rados.rados.0.smithi110.stdout:1336: writing smithi11075562-1 from 6430720 to 6922240 tid 2 2024-09-14T03:42:05.114 INFO:tasks.rados.rados.0.smithi110.stdout:1336: writing smithi11075562-1 from 6922240 to 7356416 tid 3 2024-09-14T03:42:05.118 INFO:tasks.rados.rados.0.smithi110.stdout:1336: writing smithi11075562-1 from 7356416 to 8036352 tid 4 2024-09-14T03:42:05.119 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 136 2024-09-14T03:42:05.119 INFO:tasks.rados.rados.0.smithi110.stdout:1333: finishing copy_from racing read to smithi11075562-14 2024-09-14T03:42:05.119 INFO:tasks.rados.rados.0.smithi110.stdout:1333: finishing copy_from to smithi11075562-14 2024-09-14T03:42:05.119 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 722 (ObjNum 413 snap 134 seq_num 413) dirty exists 2024-09-14T03:42:05.119 INFO:tasks.rados.rados.0.smithi110.stdout:1336: finishing write tid 1 to smithi11075562-1 2024-09-14T03:42:05.119 INFO:tasks.rados.rados.0.smithi110.stdout:1337: seq_num 425 ranges {0=720896,720896=778240,1499136=638976,2138112=188416} 2024-09-14T03:42:05.123 INFO:tasks.rados.rados.0.smithi110.stdout:1337: writing smithi11075562-27 from 0 to 720896 tid 1 2024-09-14T03:42:05.128 INFO:tasks.rados.rados.0.smithi110.stdout:1337: writing smithi11075562-27 from 720896 to 1499136 tid 2 2024-09-14T03:42:05.131 INFO:tasks.rados.rados.0.smithi110.stdout:1337: writing smithi11075562-27 from 1499136 to 2138112 tid 3 2024-09-14T03:42:05.133 INFO:tasks.rados.rados.0.smithi110.stdout:1337: writing smithi11075562-27 from 2138112 to 2326528 tid 4 2024-09-14T03:42:05.133 INFO:tasks.rados.rados.0.smithi110.stdout:1333: done (3 left) 2024-09-14T03:42:05.133 INFO:tasks.rados.rados.0.smithi110.stdout:1338: read oid 45 snap -1 2024-09-14T03:42:05.133 INFO:tasks.rados.rados.0.smithi110.stdout:1338: expect (ObjNum 406 snap 134 seq_num 406) 2024-09-14T03:42:05.133 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 136 2024-09-14T03:42:05.133 INFO:tasks.rados.rados.0.smithi110.stdout:1336: finishing write tid 2 to smithi11075562-1 2024-09-14T03:42:05.134 INFO:tasks.rados.rados.0.smithi110.stdout:1336: finishing write tid 3 to smithi11075562-1 2024-09-14T03:42:05.134 INFO:tasks.rados.rados.0.smithi110.stdout:1335: finishing copy_from racing read to smithi11075562-25 2024-09-14T03:42:05.134 INFO:tasks.rados.rados.0.smithi110.stdout:1339: seq_num 426 ranges {7372800=737280,8110080=712704,8822784=802816,9625600=90112} 2024-09-14T03:42:05.176 INFO:tasks.rados.rados.0.smithi110.stdout:1339: writing smithi11075562-13 from 7372800 to 8110080 tid 1 2024-09-14T03:42:05.180 INFO:tasks.rados.rados.0.smithi110.stdout:1339: writing smithi11075562-13 from 8110080 to 8822784 tid 2 2024-09-14T03:42:05.184 INFO:tasks.rados.rados.0.smithi110.stdout:1339: writing smithi11075562-13 from 8822784 to 9625600 tid 3 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1339: writing smithi11075562-13 from 9625600 to 9715712 tid 4 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1340: rollback oid 16 current snap is 136 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 83 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1335: finishing copy_from to smithi11075562-25 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 678 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1336: finishing write tid 4 to smithi11075562-1 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1336: finishing write tid 5 to smithi11075562-1 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1335: done (5 left) 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1341: copy_from oid 10 from oid 29 current snap is 136 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1336: finishing write tid 6 to smithi11075562-1 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 815 (ObjNum 424 snap 136 seq_num 424) dirty exists 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1336: left oid 1 (ObjNum 424 snap 136 seq_num 424) 2024-09-14T03:42:05.186 INFO:tasks.rados.rados.0.smithi110.stdout:1337: finishing write tid 1 to smithi11075562-27 2024-09-14T03:42:05.187 INFO:tasks.rados.rados.0.smithi110.stdout:1337: finishing write tid 2 to smithi11075562-27 2024-09-14T03:42:05.187 INFO:tasks.rados.rados.0.smithi110.stdout:1336: done (5 left) 2024-09-14T03:42:05.187 INFO:tasks.rados.rados.0.smithi110.stdout:1342: snap_remove snap 113 2024-09-14T03:42:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:05 smithi110 ceph-mon[86423]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:42:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:05 smithi110 ceph-mon[88262]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:42:05.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:05 smithi195 ceph-mon[82388]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:42:06.062 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:06.062 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:06.065 INFO:tasks.rados.rados.0.smithi110.stdout:1342: done (5 left) 2024-09-14T03:42:06.065 INFO:tasks.rados.rados.0.smithi110.stdout:1343: copy_from oid 12 from oid 30 current snap is 136 2024-09-14T03:42:06.066 INFO:tasks.rados.rados.0.smithi110.stdout:1337: finishing write tid 3 to smithi11075562-27 2024-09-14T03:42:06.066 INFO:tasks.rados.rados.0.smithi110.stdout:1337: finishing write tid 4 to smithi11075562-27 2024-09-14T03:42:06.066 INFO:tasks.rados.rados.0.smithi110.stdout:1337: finishing write tid 5 to smithi11075562-27 2024-09-14T03:42:06.066 INFO:tasks.rados.rados.0.smithi110.stdout:1337: finishing write tid 6 to smithi11075562-27 2024-09-14T03:42:06.066 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 142 (ObjNum 425 snap 136 seq_num 425) dirty exists 2024-09-14T03:42:06.066 INFO:tasks.rados.rados.0.smithi110.stdout:1337: left oid 27 (ObjNum 425 snap 136 seq_num 425) 2024-09-14T03:42:06.066 INFO:tasks.rados.rados.0.smithi110.stdout:1338: expect (ObjNum 406 snap 134 seq_num 406) 2024-09-14T03:42:06.189 INFO:tasks.rados.rados.0.smithi110.stdout:1339: finishing write tid 1 to smithi11075562-13 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1339: finishing write tid 2 to smithi11075562-13 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1337: done (5 left) 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1338: done (4 left) 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1344: snap_create 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1341: finishing copy_from to smithi11075562-10 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1341: got expected ENOENT (src dne) 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1341: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1340: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1340: finishing rollback tid 2 to smithi11075562-16 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 704 (ObjNum 234 snap 69 seq_num 234) dirty exists 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1339: finishing write tid 3 to smithi11075562-13 2024-09-14T03:42:06.190 INFO:tasks.rados.rados.0.smithi110.stdout:1339: finishing write tid 4 to smithi11075562-13 2024-09-14T03:42:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:1339: finishing write tid 5 to smithi11075562-13 2024-09-14T03:42:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:1339: finishing write tid 6 to smithi11075562-13 2024-09-14T03:42:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 820 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:42:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:1339: left oid 13 (ObjNum 426 snap 136 seq_num 426) 2024-09-14T03:42:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:1343: finishing copy_from racing read to smithi11075562-12 2024-09-14T03:42:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:1343: finishing copy_from to smithi11075562-12 2024-09-14T03:42:06.191 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 737 (ObjNum 404 snap 132 seq_num 404) dirty exists 2024-09-14T03:42:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:06 smithi110 ceph-mon[86423]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:42:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:06 smithi110 ceph-mon[86423]: pgmap v120: 121 pgs: 121 active+clean; 361 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 24 MiB/s, 6 objects/s recovering 2024-09-14T03:42:06.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:06 smithi110 ceph-mon[88262]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:42:06.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:06 smithi110 ceph-mon[88262]: pgmap v120: 121 pgs: 121 active+clean; 361 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 24 MiB/s, 6 objects/s recovering 2024-09-14T03:42:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:06 smithi195 ceph-mon[82388]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:42:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:06 smithi195 ceph-mon[82388]: pgmap v120: 121 pgs: 121 active+clean; 361 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 24 MiB/s, 6 objects/s recovering 2024-09-14T03:42:07.067 INFO:tasks.rados.rados.0.smithi110.stdout:1339: done (4 left) 2024-09-14T03:42:07.067 INFO:tasks.rados.rados.0.smithi110.stdout:1340: done (3 left) 2024-09-14T03:42:07.068 INFO:tasks.rados.rados.0.smithi110.stdout:1341: done (2 left) 2024-09-14T03:42:07.068 INFO:tasks.rados.rados.0.smithi110.stdout:1343: done (1 left) 2024-09-14T03:42:07.068 INFO:tasks.rados.rados.0.smithi110.stdout:1344: done (0 left) 2024-09-14T03:42:07.068 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 137 2024-09-14T03:42:07.068 INFO:tasks.rados.rados.0.smithi110.stdout:1345: seq_num 427 ranges {8372224=761856,9134080=598016,9732096=606208,10338304=401408} 2024-09-14T03:42:07.116 INFO:tasks.rados.rados.0.smithi110.stdout:1345: writing smithi11075562-11 from 8372224 to 9134080 tid 1 2024-09-14T03:42:07.119 INFO:tasks.rados.rados.0.smithi110.stdout:1345: writing smithi11075562-11 from 9134080 to 9732096 tid 2 2024-09-14T03:42:07.123 INFO:tasks.rados.rados.0.smithi110.stdout:1345: writing smithi11075562-11 from 9732096 to 10338304 tid 3 2024-09-14T03:42:07.125 INFO:tasks.rados.rados.0.smithi110.stdout:1345: writing smithi11075562-11 from 10338304 to 10739712 tid 4 2024-09-14T03:42:07.125 INFO:tasks.rados.rados.0.smithi110.stdout:1346: rmattr oid 30 current snap is 137 2024-09-14T03:42:07.125 INFO:tasks.rados.rados.0.smithi110.stdout:1346: done (1 left) 2024-09-14T03:42:07.125 INFO:tasks.rados.rados.0.smithi110.stdout:1347: delete oid 19 current snap is 137 2024-09-14T03:42:07.126 INFO:tasks.rados.rados.0.smithi110.stdout:1345: finishing write tid 1 to smithi11075562-11 2024-09-14T03:42:07.129 INFO:tasks.rados.rados.0.smithi110.stdout:1345: finishing write tid 2 to smithi11075562-11 2024-09-14T03:42:07.129 INFO:tasks.rados.rados.0.smithi110.stdout:1345: finishing write tid 3 to smithi11075562-11 2024-09-14T03:42:07.130 INFO:tasks.rados.rados.0.smithi110.stdout:1345: finishing write tid 4 to smithi11075562-11 2024-09-14T03:42:07.131 INFO:tasks.rados.rados.0.smithi110.stdout:1347: done (1 left) 2024-09-14T03:42:07.131 INFO:tasks.rados.rados.0.smithi110.stdout:1348: rollback oid 30 current snap is 137 2024-09-14T03:42:07.131 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 121 2024-09-14T03:42:07.131 INFO:tasks.rados.rados.0.smithi110.stdout:1345: finishing write tid 5 to smithi11075562-11 2024-09-14T03:42:07.131 INFO:tasks.rados.rados.0.smithi110.stdout:1349: copy_from oid 23 from oid 17 current snap is 137 2024-09-14T03:42:07.131 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 137 2024-09-14T03:42:07.131 INFO:tasks.rados.rados.0.smithi110.stdout:1350: seq_num 429 ranges {4120576=794624,4915200=770048,5685248=622592,6307840=212992} 2024-09-14T03:42:07.157 INFO:tasks.rados.rados.0.smithi110.stdout:1350: writing smithi11075562-24 from 4120576 to 4915200 tid 1 2024-09-14T03:42:07.161 INFO:tasks.rados.rados.0.smithi110.stdout:1350: writing smithi11075562-24 from 4915200 to 5685248 tid 2 2024-09-14T03:42:07.165 INFO:tasks.rados.rados.0.smithi110.stdout:1350: writing smithi11075562-24 from 5685248 to 6307840 tid 3 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1350: writing smithi11075562-24 from 6307840 to 6520832 tid 4 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1351: read oid 34 snap 136 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1351: expect (ObjNum 387 snap 127 seq_num 387) 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1352: snap_create 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1345: finishing write tid 6 to smithi11075562-11 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 637 (ObjNum 427 snap 137 seq_num 427) dirty exists 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1345: left oid 11 (ObjNum 427 snap 137 seq_num 427) 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1349: finishing copy_from to smithi11075562-23 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1349: got expected ENOENT (src dne) 2024-09-14T03:42:07.167 INFO:tasks.rados.rados.0.smithi110.stdout:1349: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:42:07.168 INFO:tasks.rados.rados.0.smithi110.stdout:1348: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:42:07.168 INFO:tasks.rados.rados.0.smithi110.stdout:1348: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:42:07.168 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 800 (ObjNum 97 snap 0 seq_num 0) dirty dne 2024-09-14T03:42:07.168 INFO:tasks.rados.rados.0.smithi110.stdout:1350: finishing write tid 1 to smithi11075562-24 2024-09-14T03:42:07.168 INFO:tasks.rados.rados.0.smithi110.stdout:1350: finishing write tid 2 to smithi11075562-24 2024-09-14T03:42:07.168 INFO:tasks.rados.rados.0.smithi110.stdout:1350: finishing write tid 3 to smithi11075562-24 2024-09-14T03:42:07.169 INFO:tasks.rados.rados.0.smithi110.stdout:1350: finishing write tid 4 to smithi11075562-24 2024-09-14T03:42:07.169 INFO:tasks.rados.rados.0.smithi110.stdout:1350: finishing write tid 5 to smithi11075562-24 2024-09-14T03:42:07.170 INFO:tasks.rados.rados.0.smithi110.stdout:1350: finishing write tid 6 to smithi11075562-24 2024-09-14T03:42:07.170 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 510 (ObjNum 429 snap 137 seq_num 429) dirty exists 2024-09-14T03:42:07.170 INFO:tasks.rados.rados.0.smithi110.stdout:1350: left oid 24 (ObjNum 429 snap 137 seq_num 429) 2024-09-14T03:42:07.179 INFO:tasks.rados.rados.0.smithi110.stdout:1351: expect (ObjNum 387 snap 127 seq_num 387) 2024-09-14T03:42:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:07 smithi195 ceph-mon[82388]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/858 objects degraded (3.030%), 5 pgs degraded) 2024-09-14T03:42:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:07 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:42:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:07 smithi195 ceph-mon[82388]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:42:07.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:07 smithi110 ceph-mon[88262]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/858 objects degraded (3.030%), 5 pgs degraded) 2024-09-14T03:42:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:07 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:42:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:07 smithi110 ceph-mon[88262]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:42:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:07 smithi110 ceph-mon[86423]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/858 objects degraded (3.030%), 5 pgs degraded) 2024-09-14T03:42:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:07 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:42:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:07 smithi110 ceph-mon[86423]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:42:08.076 INFO:tasks.rados.rados.0.smithi110.stdout:1345: done (5 left) 2024-09-14T03:42:08.076 INFO:tasks.rados.rados.0.smithi110.stdout:1348: done (4 left) 2024-09-14T03:42:08.076 INFO:tasks.rados.rados.0.smithi110.stdout:1349: done (3 left) 2024-09-14T03:42:08.076 INFO:tasks.rados.rados.0.smithi110.stdout:1350: done (2 left) 2024-09-14T03:42:08.076 INFO:tasks.rados.rados.0.smithi110.stdout:1351: done (1 left) 2024-09-14T03:42:08.076 INFO:tasks.rados.rados.0.smithi110.stdout:1352: done (0 left) 2024-09-14T03:42:08.076 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 138 2024-09-14T03:42:08.076 INFO:tasks.rados.rados.0.smithi110.stdout:1353: seq_num 430 ranges {581632=8192} 2024-09-14T03:42:08.079 INFO:tasks.rados.rados.0.smithi110.stdout:1353: writing smithi11075562-22 from 581632 to 589824 tid 1 2024-09-14T03:42:08.079 INFO:tasks.rados.rados.0.smithi110.stdout:1354: copy_from oid 27 from oid 25 current snap is 138 2024-09-14T03:42:08.079 INFO:tasks.rados.rados.0.smithi110.stdout:1355: read oid 38 snap -1 2024-09-14T03:42:08.079 INFO:tasks.rados.rados.0.smithi110.stdout:1355: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:42:08.080 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 138 2024-09-14T03:42:08.080 INFO:tasks.rados.rados.0.smithi110.stdout:1356: seq_num 431 ranges {0=24576} 2024-09-14T03:42:08.080 INFO:tasks.rados.rados.0.smithi110.stdout:1356: writing smithi11075562-29 from 0 to 24576 tid 1 2024-09-14T03:42:08.080 INFO:tasks.rados.rados.0.smithi110.stdout:1357: copy_from oid 32 from oid 33 current snap is 138 2024-09-14T03:42:08.080 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 138 2024-09-14T03:42:08.080 INFO:tasks.rados.rados.0.smithi110.stdout:1358: seq_num 432 ranges {6520832=40960} 2024-09-14T03:42:08.116 INFO:tasks.rados.rados.0.smithi110.stdout:1358: writing smithi11075562-24 from 6520832 to 6561792 tid 1 2024-09-14T03:42:08.116 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 138 2024-09-14T03:42:08.117 INFO:tasks.rados.rados.0.smithi110.stdout:1353: finishing write tid 1 to smithi11075562-22 2024-09-14T03:42:08.117 INFO:tasks.rados.rados.0.smithi110.stdout:1359: seq_num 433 ranges {0=57344} 2024-09-14T03:42:08.117 INFO:tasks.rados.rados.0.smithi110.stdout:1359: writing smithi11075562-49 from 0 to 57344 tid 1 2024-09-14T03:42:08.117 INFO:tasks.rados.rados.0.smithi110.stdout:1360: read oid 42 snap -1 2024-09-14T03:42:08.117 INFO:tasks.rados.rados.0.smithi110.stdout:1360: expect deleted 2024-09-14T03:42:08.117 INFO:tasks.rados.rados.0.smithi110.stdout:1361: snap_create 2024-09-14T03:42:08.117 INFO:tasks.rados.rados.0.smithi110.stdout:1355: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:42:08.133 INFO:tasks.rados.rados.0.smithi110.stdout:1356: finishing write tid 1 to smithi11075562-29 2024-09-14T03:42:08.133 INFO:tasks.rados.rados.0.smithi110.stdout:1356: finishing write tid 2 to smithi11075562-29 2024-09-14T03:42:08.133 INFO:tasks.rados.rados.0.smithi110.stdout:1353: finishing write tid 2 to smithi11075562-22 2024-09-14T03:42:08.133 INFO:tasks.rados.rados.0.smithi110.stdout:1356: finishing write tid 3 to smithi11075562-29 2024-09-14T03:42:08.133 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 480 (ObjNum 431 snap 138 seq_num 431) dirty exists 2024-09-14T03:42:08.133 INFO:tasks.rados.rados.0.smithi110.stdout:1356: left oid 29 (ObjNum 431 snap 138 seq_num 431) 2024-09-14T03:42:08.133 INFO:tasks.rados.rados.0.smithi110.stdout:1353: finishing write tid 3 to smithi11075562-22 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 803 (ObjNum 430 snap 138 seq_num 430) dirty exists 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1353: left oid 22 (ObjNum 430 snap 138 seq_num 430) 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1357: finishing copy_from to smithi11075562-32 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1357: got expected ENOENT (src dne) 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1357: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1354: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1358: finishing write tid 1 to smithi11075562-24 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1358: finishing write tid 2 to smithi11075562-24 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1359: finishing write tid 1 to smithi11075562-49 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1359: finishing write tid 2 to smithi11075562-49 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1354: finishing copy_from to smithi11075562-27 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 678 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:1358: finishing write tid 3 to smithi11075562-24 2024-09-14T03:42:08.134 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 512 (ObjNum 432 snap 138 seq_num 432) dirty exists 2024-09-14T03:42:08.135 INFO:tasks.rados.rados.0.smithi110.stdout:1358: left oid 24 (ObjNum 432 snap 138 seq_num 432) 2024-09-14T03:42:08.135 INFO:tasks.rados.rados.0.smithi110.stdout:1359: finishing write tid 3 to smithi11075562-49 2024-09-14T03:42:08.135 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 828 (ObjNum 433 snap 138 seq_num 433) dirty exists 2024-09-14T03:42:08.135 INFO:tasks.rados.rados.0.smithi110.stdout:1359: left oid 49 (ObjNum 433 snap 138 seq_num 433) 2024-09-14T03:42:08.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:08 smithi110 ceph-mon[88262]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:42:08.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:08 smithi110 ceph-mon[88262]: pgmap v123: 121 pgs: 121 active+clean; 390 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.7 MiB/s wr, 23 op/s; 5.2 MiB/s, 1 objects/s recovering 2024-09-14T03:42:08.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:08 smithi110 ceph-mon[86423]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:42:08.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:08 smithi110 ceph-mon[86423]: pgmap v123: 121 pgs: 121 active+clean; 390 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.7 MiB/s wr, 23 op/s; 5.2 MiB/s, 1 objects/s recovering 2024-09-14T03:42:08.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:08 smithi195 ceph-mon[82388]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:42:08.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:08 smithi195 ceph-mon[82388]: pgmap v123: 121 pgs: 121 active+clean; 390 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.7 MiB/s wr, 23 op/s; 5.2 MiB/s, 1 objects/s recovering 2024-09-14T03:42:08.679 INFO:tasks.rados.rados.0.smithi110.stdout:1353: done (8 left) 2024-09-14T03:42:08.679 INFO:tasks.rados.rados.0.smithi110.stdout:1354: done (7 left) 2024-09-14T03:42:08.679 INFO:tasks.rados.rados.0.smithi110.stdout:1355: done (6 left) 2024-09-14T03:42:08.679 INFO:tasks.rados.rados.0.smithi110.stdout:1356: done (5 left) 2024-09-14T03:42:08.679 INFO:tasks.rados.rados.0.smithi110.stdout:1357: done (4 left) 2024-09-14T03:42:08.680 INFO:tasks.rados.rados.0.smithi110.stdout:1358: done (3 left) 2024-09-14T03:42:08.680 INFO:tasks.rados.rados.0.smithi110.stdout:1359: done (2 left) 2024-09-14T03:42:08.680 INFO:tasks.rados.rados.0.smithi110.stdout:1360: done (1 left) 2024-09-14T03:42:08.680 INFO:tasks.rados.rados.0.smithi110.stdout:1361: done (0 left) 2024-09-14T03:42:08.680 INFO:tasks.rados.rados.0.smithi110.stdout:1362: setattr oid 23 current snap is 139 2024-09-14T03:42:08.680 INFO:tasks.rados.rados.0.smithi110.stdout:1363: snap_remove snap 137 2024-09-14T03:42:09.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:09 smithi110 ceph-mon[88262]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:42:09.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:09 smithi110 ceph-mon[88262]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:42:09.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:09 smithi110 ceph-mon[86423]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:42:09.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:09 smithi110 ceph-mon[86423]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:42:09.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:09 smithi195 ceph-mon[82388]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:42:09.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:09 smithi195 ceph-mon[82388]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:42:09.681 INFO:tasks.rados.rados.0.smithi110.stdout:1363: done (1 left) 2024-09-14T03:42:09.681 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 139 2024-09-14T03:42:09.681 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 513 (ObjNum 371 snap 122 seq_num 371) dirty exists 2024-09-14T03:42:09.681 INFO:tasks.rados.rados.0.smithi110.stdout:1364: seq_num 434 ranges {3514368=73728} 2024-09-14T03:42:09.700 INFO:tasks.rados.rados.0.smithi110.stdout:1364: writing smithi11075562-45 from 3514368 to 3588096 tid 1 2024-09-14T03:42:09.700 INFO:tasks.rados.rados.0.smithi110.stdout:1362: done (1 left) 2024-09-14T03:42:09.700 INFO:tasks.rados.rados.0.smithi110.stdout:1365: snap_remove snap 78 2024-09-14T03:42:10.682 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:10.686 INFO:tasks.rados.rados.0.smithi110.stdout:1364: finishing write tid 1 to smithi11075562-45 2024-09-14T03:42:10.686 INFO:tasks.rados.rados.0.smithi110.stdout:1364: finishing write tid 2 to smithi11075562-45 2024-09-14T03:42:10.686 INFO:tasks.rados.rados.0.smithi110.stdout:1364: finishing write tid 3 to smithi11075562-45 2024-09-14T03:42:10.687 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 917 (ObjNum 434 snap 139 seq_num 434) dirty exists 2024-09-14T03:42:10.687 INFO:tasks.rados.rados.0.smithi110.stdout:1364: left oid 45 (ObjNum 434 snap 139 seq_num 434) 2024-09-14T03:42:10.687 INFO:tasks.rados.rados.0.smithi110.stdout:1364: done (1 left) 2024-09-14T03:42:10.687 INFO:tasks.rados.rados.0.smithi110.stdout:1365: done (0 left) 2024-09-14T03:42:10.687 INFO:tasks.rados.rados.0.smithi110.stdout:1366: delete oid 3 current snap is 139 2024-09-14T03:42:10.701 INFO:tasks.rados.rados.0.smithi110.stdout:1366: done (0 left) 2024-09-14T03:42:10.701 INFO:tasks.rados.rados.0.smithi110.stdout:1367: rollback oid 50 current snap is 139 2024-09-14T03:42:10.701 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 83 2024-09-14T03:42:10.701 INFO:tasks.rados.rados.0.smithi110.stdout:1368: copy_from oid 39 from oid 10 current snap is 139 2024-09-14T03:42:10.701 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 139 2024-09-14T03:42:10.701 INFO:tasks.rados.rados.0.smithi110.stdout:1369: seq_num 436 ranges {6709248=106496} 2024-09-14T03:42:10.740 INFO:tasks.rados.rados.0.smithi110.stdout:1369: writing smithi11075562-2 from 6709248 to 6815744 tid 1 2024-09-14T03:42:10.740 INFO:tasks.rados.rados.0.smithi110.stdout:1370: read oid 36 snap -1 2024-09-14T03:42:10.740 INFO:tasks.rados.rados.0.smithi110.stdout:1370: expect deleted 2024-09-14T03:42:10.740 INFO:tasks.rados.rados.0.smithi110.stdout:1371: rollback oid 32 current snap is 139 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 110 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1367: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1367: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1367: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 135 (ObjNum 261 snap 79 seq_num 261) dirty exists 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1368: finishing copy_from racing read to smithi11075562-39 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1367: done (4 left) 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1372: rollback oid 25 current snap is 139 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 25 to 138 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1368: finishing copy_from to smithi11075562-39 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 502 (ObjNum 389 snap 129 seq_num 389) dirty exists 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1368: done (4 left) 2024-09-14T03:42:10.741 INFO:tasks.rados.rados.0.smithi110.stdout:1373: copy_from oid 38 from oid 9 current snap is 139 2024-09-14T03:42:10.742 INFO:tasks.rados.rados.0.smithi110.stdout:1374: snap_create 2024-09-14T03:42:10.742 INFO:tasks.rados.rados.0.smithi110.stdout:1373: finishing copy_from to smithi11075562-38 2024-09-14T03:42:10.742 INFO:tasks.rados.rados.0.smithi110.stdout:1373: got expected ENOENT (src dne) 2024-09-14T03:42:10.742 INFO:tasks.rados.rados.0.smithi110.stdout:1373: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:42:10.743 INFO:tasks.rados.rados.0.smithi110.stdout:1369: finishing write tid 1 to smithi11075562-2 2024-09-14T03:42:10.743 INFO:tasks.rados.rados.0.smithi110.stdout:1369: finishing write tid 2 to smithi11075562-2 2024-09-14T03:42:10.743 INFO:tasks.rados.rados.0.smithi110.stdout:1371: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:42:10.743 INFO:tasks.rados.rados.0.smithi110.stdout:1371: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:42:10.744 INFO:tasks.rados.rados.0.smithi110.stdout:1369: finishing write tid 3 to smithi11075562-2 2024-09-14T03:42:10.744 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 681 (ObjNum 436 snap 139 seq_num 436) dirty exists 2024-09-14T03:42:10.744 INFO:tasks.rados.rados.0.smithi110.stdout:1369: left oid 2 (ObjNum 436 snap 139 seq_num 436) 2024-09-14T03:42:10.744 INFO:tasks.rados.rados.0.smithi110.stdout:1371: finishing rollback tid 2 to smithi11075562-32 2024-09-14T03:42:10.744 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 819 (ObjNum 335 snap 106 seq_num 335) dirty exists 2024-09-14T03:42:10.745 INFO:tasks.rados.rados.0.smithi110.stdout:1372: finishing rollback tid 0 to smithi11075562-25 2024-09-14T03:42:10.745 INFO:tasks.rados.rados.0.smithi110.stdout:1372: finishing rollback tid 1 to smithi11075562-25 2024-09-14T03:42:10.745 INFO:tasks.rados.rados.0.smithi110.stdout:1372: finishing rollback tid 2 to smithi11075562-25 2024-09-14T03:42:10.745 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 929 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:10.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:10 smithi110 ceph-mon[88262]: osdmap e334: 8 total, 8 up, 8 in 2024-09-14T03:42:10.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:10 smithi110 ceph-mon[88262]: pgmap v127: 121 pgs: 121 active+clean; 400 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 10 MiB/s wr, 24 op/s 2024-09-14T03:42:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:10 smithi110 ceph-mon[86423]: osdmap e334: 8 total, 8 up, 8 in 2024-09-14T03:42:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:10 smithi110 ceph-mon[86423]: pgmap v127: 121 pgs: 121 active+clean; 400 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 10 MiB/s wr, 24 op/s 2024-09-14T03:42:10.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:10 smithi195 ceph-mon[82388]: osdmap e334: 8 total, 8 up, 8 in 2024-09-14T03:42:10.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:10 smithi195 ceph-mon[82388]: pgmap v127: 121 pgs: 121 active+clean; 400 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 10 MiB/s wr, 24 op/s 2024-09-14T03:42:11.688 INFO:tasks.rados.rados.0.smithi110.stdout:1369: done (5 left) 2024-09-14T03:42:11.688 INFO:tasks.rados.rados.0.smithi110.stdout:1370: done (4 left) 2024-09-14T03:42:11.689 INFO:tasks.rados.rados.0.smithi110.stdout:1371: done (3 left) 2024-09-14T03:42:11.689 INFO:tasks.rados.rados.0.smithi110.stdout:1372: done (2 left) 2024-09-14T03:42:11.689 INFO:tasks.rados.rados.0.smithi110.stdout:1373: done (1 left) 2024-09-14T03:42:11.689 INFO:tasks.rados.rados.0.smithi110.stdout:1374: done (0 left) 2024-09-14T03:42:11.689 INFO:tasks.rados.rados.0.smithi110.stdout:1375: read oid 31 snap 90 2024-09-14T03:42:11.689 INFO:tasks.rados.rados.0.smithi110.stdout:1375: expect deleted 2024-09-14T03:42:11.689 INFO:tasks.rados.rados.0.smithi110.stdout:1376: delete oid 4 current snap is 140 2024-09-14T03:42:11.697 INFO:tasks.rados.rados.0.smithi110.stdout:1375: done (1 left) 2024-09-14T03:42:11.697 INFO:tasks.rados.rados.0.smithi110.stdout:1376: done (0 left) 2024-09-14T03:42:11.697 INFO:tasks.rados.rados.0.smithi110.stdout:1377: snap_remove snap 106 2024-09-14T03:42:11.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:42:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:42:11] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-14T03:42:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:11 smithi110 ceph-mon[88262]: osdmap e335: 8 total, 8 up, 8 in 2024-09-14T03:42:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:11 smithi110 ceph-mon[86423]: osdmap e335: 8 total, 8 up, 8 in 2024-09-14T03:42:11.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:11 smithi195 ceph-mon[82388]: osdmap e335: 8 total, 8 up, 8 in 2024-09-14T03:42:12.698 INFO:tasks.rados.rados.0.smithi110.stdout:1377: done (0 left) 2024-09-14T03:42:12.698 INFO:tasks.rados.rados.0.smithi110.stdout:1378: read oid 34 snap -1 2024-09-14T03:42:12.699 INFO:tasks.rados.rados.0.smithi110.stdout:1378: expect (ObjNum 387 snap 127 seq_num 387) 2024-09-14T03:42:12.699 INFO:tasks.rados.rados.0.smithi110.stdout:1379: snap_create 2024-09-14T03:42:12.726 INFO:tasks.rados.rados.0.smithi110.stdout:1378: expect (ObjNum 387 snap 127 seq_num 387) 2024-09-14T03:42:12.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[88262]: osdmap e336: 8 total, 8 up, 8 in 2024-09-14T03:42:12.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[88262]: pgmap v130: 121 pgs: 121 active+clean; 400 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 560 B/s rd, 2.9 MiB/s wr, 3 op/s 2024-09-14T03:42:12.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:12.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:12.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:12.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[86423]: osdmap e336: 8 total, 8 up, 8 in 2024-09-14T03:42:12.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[86423]: pgmap v130: 121 pgs: 121 active+clean; 400 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 560 B/s rd, 2.9 MiB/s wr, 3 op/s 2024-09-14T03:42:12.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:12.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:12.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:12.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:12 smithi195 ceph-mon[82388]: osdmap e336: 8 total, 8 up, 8 in 2024-09-14T03:42:12.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:12 smithi195 ceph-mon[82388]: pgmap v130: 121 pgs: 121 active+clean; 400 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 560 B/s rd, 2.9 MiB/s wr, 3 op/s 2024-09-14T03:42:12.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:12 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:12.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:12 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:12.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:13.680 INFO:tasks.rados.rados.0.smithi110.stdout:1378: done (1 left) 2024-09-14T03:42:13.680 INFO:tasks.rados.rados.0.smithi110.stdout:1379: done (0 left) 2024-09-14T03:42:13.680 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 141 2024-09-14T03:42:13.680 INFO:tasks.rados.rados.0.smithi110.stdout:1380: seq_num 438 ranges {1531904=139264} 2024-09-14T03:42:13.689 INFO:tasks.rados.rados.0.smithi110.stdout:1380: writing smithi11075562-16 from 1531904 to 1671168 tid 1 2024-09-14T03:42:13.689 INFO:tasks.rados.rados.0.smithi110.stdout:1381: snap_create 2024-09-14T03:42:13.692 INFO:tasks.rados.rados.0.smithi110.stdout:1380: finishing write tid 1 to smithi11075562-16 2024-09-14T03:42:13.693 INFO:tasks.rados.rados.0.smithi110.stdout:1380: finishing write tid 2 to smithi11075562-16 2024-09-14T03:42:13.694 INFO:tasks.rados.rados.0.smithi110.stdout:1380: finishing write tid 3 to smithi11075562-16 2024-09-14T03:42:13.695 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 735 (ObjNum 438 snap 141 seq_num 438) dirty exists 2024-09-14T03:42:13.695 INFO:tasks.rados.rados.0.smithi110.stdout:1380: left oid 16 (ObjNum 438 snap 141 seq_num 438) 2024-09-14T03:42:13.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:13 smithi110 ceph-mon[88262]: osdmap e337: 8 total, 8 up, 8 in 2024-09-14T03:42:13.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:13 smithi110 ceph-mon[88262]: osdmap e338: 8 total, 8 up, 8 in 2024-09-14T03:42:13.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:13 smithi110 ceph-mon[86423]: osdmap e337: 8 total, 8 up, 8 in 2024-09-14T03:42:13.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:13 smithi110 ceph-mon[86423]: osdmap e338: 8 total, 8 up, 8 in 2024-09-14T03:42:13.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:13 smithi195 ceph-mon[82388]: osdmap e337: 8 total, 8 up, 8 in 2024-09-14T03:42:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:13 smithi195 ceph-mon[82388]: osdmap e338: 8 total, 8 up, 8 in 2024-09-14T03:42:14.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:14.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:42:14.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:14.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:14.683 INFO:tasks.rados.rados.0.smithi110.stdout:1380: done (1 left) 2024-09-14T03:42:14.683 INFO:tasks.rados.rados.0.smithi110.stdout:1381: done (0 left) 2024-09-14T03:42:14.683 INFO:tasks.rados.rados.0.smithi110.stdout:1382: read oid 39 snap -1 2024-09-14T03:42:14.683 INFO:tasks.rados.rados.0.smithi110.stdout:1382: expect (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:42:14.683 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 142 2024-09-14T03:42:14.683 INFO:tasks.rados.rados.0.smithi110.stdout:1383: seq_num 439 ranges {0=155648} 2024-09-14T03:42:14.684 INFO:tasks.rados.rados.0.smithi110.stdout:1383: writing smithi11075562-31 from 0 to 155648 tid 1 2024-09-14T03:42:14.684 INFO:tasks.rados.rados.0.smithi110.stdout:1384: read oid 20 snap -1 2024-09-14T03:42:14.684 INFO:tasks.rados.rados.0.smithi110.stdout:1384: expect (ObjNum 292 snap 90 seq_num 292) 2024-09-14T03:42:14.684 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 142 2024-09-14T03:42:14.684 INFO:tasks.rados.rados.0.smithi110.stdout:1385: seq_num 440 ranges {1982464=172032} 2024-09-14T03:42:14.696 INFO:tasks.rados.rados.0.smithi110.stdout:1385: writing smithi11075562-50 from 1982464 to 2154496 tid 1 2024-09-14T03:42:14.696 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 142 2024-09-14T03:42:14.696 INFO:tasks.rados.rados.0.smithi110.stdout:1383: finishing write tid 1 to smithi11075562-31 2024-09-14T03:42:14.696 INFO:tasks.rados.rados.0.smithi110.stdout:1383: finishing write tid 2 to smithi11075562-31 2024-09-14T03:42:14.696 INFO:tasks.rados.rados.0.smithi110.stdout:1384: expect (ObjNum 292 snap 90 seq_num 292) 2024-09-14T03:42:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:1383: finishing write tid 3 to smithi11075562-31 2024-09-14T03:42:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 502 (ObjNum 439 snap 142 seq_num 439) dirty exists 2024-09-14T03:42:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:1383: left oid 31 (ObjNum 439 snap 142 seq_num 439) 2024-09-14T03:42:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:1386: seq_num 441 ranges {4030464=188416} 2024-09-14T03:42:14.726 INFO:tasks.rados.rados.0.smithi110.stdout:1386: writing smithi11075562-40 from 4030464 to 4218880 tid 1 2024-09-14T03:42:14.726 INFO:tasks.rados.rados.0.smithi110.stdout:1383: done (4 left) 2024-09-14T03:42:14.726 INFO:tasks.rados.rados.0.smithi110.stdout:1384: done (3 left) 2024-09-14T03:42:14.726 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 142 2024-09-14T03:42:14.726 INFO:tasks.rados.rados.0.smithi110.stdout:1385: finishing write tid 1 to smithi11075562-50 2024-09-14T03:42:14.727 INFO:tasks.rados.rados.0.smithi110.stdout:1387: seq_num 442 ranges {2482176=204800} 2024-09-14T03:42:14.740 INFO:tasks.rados.rados.0.smithi110.stdout:1387: writing smithi11075562-21 from 2482176 to 2686976 tid 1 2024-09-14T03:42:14.740 INFO:tasks.rados.rados.0.smithi110.stdout:1388: snap_remove snap 127 2024-09-14T03:42:14.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:14 smithi195 ceph-mon[82388]: pgmap v133: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 396 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-14T03:42:14.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:14 smithi195 ceph-mon[82388]: osdmap e339: 8 total, 8 up, 8 in 2024-09-14T03:42:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:14 smithi110 ceph-mon[88262]: pgmap v133: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 396 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-14T03:42:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:14 smithi110 ceph-mon[88262]: osdmap e339: 8 total, 8 up, 8 in 2024-09-14T03:42:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:14 smithi110 ceph-mon[86423]: pgmap v133: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 396 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-14T03:42:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:14 smithi110 ceph-mon[86423]: osdmap e339: 8 total, 8 up, 8 in 2024-09-14T03:42:15.709 INFO:tasks.rados.rados.0.smithi110.stdout:1388: done (4 left) 2024-09-14T03:42:15.709 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 142 2024-09-14T03:42:15.709 INFO:tasks.rados.rados.0.smithi110.stdout:1385: finishing write tid 2 to smithi11075562-50 2024-09-14T03:42:15.709 INFO:tasks.rados.rados.0.smithi110.stdout:1385: finishing write tid 3 to smithi11075562-50 2024-09-14T03:42:15.709 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 140 (ObjNum 440 snap 142 seq_num 440) dirty exists 2024-09-14T03:42:15.709 INFO:tasks.rados.rados.0.smithi110.stdout:1385: left oid 50 (ObjNum 440 snap 142 seq_num 440) 2024-09-14T03:42:15.709 INFO:tasks.rados.rados.0.smithi110.stdout:1382: expect (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:42:15.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:15 smithi110 ceph-mon[88262]: mgrmap e44: y(active, since 93s), standbys: x 2024-09-14T03:42:15.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:15 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:42:15.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:15 smithi110 ceph-mon[86423]: mgrmap e44: y(active, since 93s), standbys: x 2024-09-14T03:42:15.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:15 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:42:16.101 INFO:tasks.rados.rados.0.smithi110.stdout:1386: finishing write tid 1 to smithi11075562-40 2024-09-14T03:42:16.101 INFO:tasks.rados.rados.0.smithi110.stdout:1386: finishing write tid 2 to smithi11075562-40 2024-09-14T03:42:16.101 INFO:tasks.rados.rados.0.smithi110.stdout:1386: finishing write tid 3 to smithi11075562-40 2024-09-14T03:42:16.101 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 738 (ObjNum 441 snap 142 seq_num 441) dirty exists 2024-09-14T03:42:16.101 INFO:tasks.rados.rados.0.smithi110.stdout:1386: left oid 40 (ObjNum 441 snap 142 seq_num 441) 2024-09-14T03:42:16.101 INFO:tasks.rados.rados.0.smithi110.stdout:1387: finishing write tid 1 to smithi11075562-21 2024-09-14T03:42:16.101 INFO:tasks.rados.rados.0.smithi110.stdout:1387: finishing write tid 2 to smithi11075562-21 2024-09-14T03:42:16.102 INFO:tasks.rados.rados.0.smithi110.stdout:1387: finishing write tid 3 to smithi11075562-21 2024-09-14T03:42:16.102 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 798 (ObjNum 442 snap 142 seq_num 442) dirty exists 2024-09-14T03:42:16.102 INFO:tasks.rados.rados.0.smithi110.stdout:1387: left oid 21 (ObjNum 442 snap 142 seq_num 442) 2024-09-14T03:42:16.102 INFO:tasks.rados.rados.0.smithi110.stdout:1389: seq_num 443 ranges {4038656=221184} 2024-09-14T03:42:16.123 INFO:tasks.rados.rados.0.smithi110.stdout:1389: writing smithi11075562-14 from 4038656 to 4259840 tid 1 2024-09-14T03:42:16.123 INFO:tasks.rados.rados.0.smithi110.stdout:1382: done (4 left) 2024-09-14T03:42:16.123 INFO:tasks.rados.rados.0.smithi110.stdout:1385: done (3 left) 2024-09-14T03:42:16.123 INFO:tasks.rados.rados.0.smithi110.stdout:1386: done (2 left) 2024-09-14T03:42:16.123 INFO:tasks.rados.rados.0.smithi110.stdout:1387: done (1 left) 2024-09-14T03:42:16.123 INFO:tasks.rados.rados.0.smithi110.stdout:1390: setattr oid 47 current snap is 142 2024-09-14T03:42:16.124 INFO:tasks.rados.rados.0.smithi110.stdout:1391: copy_from oid 46 from oid 9 current snap is 142 2024-09-14T03:42:16.124 INFO:tasks.rados.rados.0.smithi110.stdout:1392: snap_create 2024-09-14T03:42:16.125 INFO:tasks.rados.rados.0.smithi110.stdout:1391: finishing copy_from to smithi11075562-46 2024-09-14T03:42:16.125 INFO:tasks.rados.rados.0.smithi110.stdout:1391: got expected ENOENT (src dne) 2024-09-14T03:42:16.126 INFO:tasks.rados.rados.0.smithi110.stdout:1391: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:42:16.126 INFO:tasks.rados.rados.0.smithi110.stdout:1389: finishing write tid 1 to smithi11075562-14 2024-09-14T03:42:16.126 INFO:tasks.rados.rados.0.smithi110.stdout:1389: finishing write tid 2 to smithi11075562-14 2024-09-14T03:42:16.127 INFO:tasks.rados.rados.0.smithi110.stdout:1389: finishing write tid 3 to smithi11075562-14 2024-09-14T03:42:16.127 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 807 (ObjNum 443 snap 142 seq_num 443) dirty exists 2024-09-14T03:42:16.127 INFO:tasks.rados.rados.0.smithi110.stdout:1389: left oid 14 (ObjNum 443 snap 142 seq_num 443) 2024-09-14T03:42:16.128 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 947 (ObjNum 305 snap 98 seq_num 305) dirty exists 2024-09-14T03:42:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:15 smithi195 ceph-mon[82388]: mgrmap e44: y(active, since 93s), standbys: x 2024-09-14T03:42:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:15 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:42:16.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:16.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:16.718 INFO:tasks.rados.rados.0.smithi110.stdout:1389: done (3 left) 2024-09-14T03:42:16.718 INFO:tasks.rados.rados.0.smithi110.stdout:1390: done (2 left) 2024-09-14T03:42:16.718 INFO:tasks.rados.rados.0.smithi110.stdout:1391: done (1 left) 2024-09-14T03:42:16.718 INFO:tasks.rados.rados.0.smithi110.stdout:1392: done (0 left) 2024-09-14T03:42:16.719 INFO:tasks.rados.rados.0.smithi110.stdout:1393: snap_remove snap 134 2024-09-14T03:42:17.192 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:16 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:42:17.192 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:16 smithi110 ceph-mon[86423]: Upgrade: osd.1 is safe to restart 2024-09-14T03:42:17.192 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:16 smithi110 ceph-mon[86423]: osdmap e340: 8 total, 8 up, 8 in 2024-09-14T03:42:17.192 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:16 smithi110 ceph-mon[86423]: pgmap v136: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 394 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-14T03:42:17.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:16 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:42:17.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:16 smithi110 ceph-mon[88262]: Upgrade: osd.1 is safe to restart 2024-09-14T03:42:17.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:16 smithi110 ceph-mon[88262]: osdmap e340: 8 total, 8 up, 8 in 2024-09-14T03:42:17.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:16 smithi110 ceph-mon[88262]: pgmap v136: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 394 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-14T03:42:17.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:16 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:42:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:16 smithi195 ceph-mon[82388]: Upgrade: osd.1 is safe to restart 2024-09-14T03:42:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:16 smithi195 ceph-mon[82388]: osdmap e340: 8 total, 8 up, 8 in 2024-09-14T03:42:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:16 smithi195 ceph-mon[82388]: pgmap v136: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 394 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-14T03:42:17.719 INFO:tasks.rados.rados.0.smithi110.stdout:1393: done (0 left) 2024-09-14T03:42:17.720 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 143 2024-09-14T03:42:17.720 INFO:tasks.rados.rados.0.smithi110.stdout:1394: seq_num 444 ranges {2211840=237568} 2024-09-14T03:42:17.734 INFO:tasks.rados.rados.0.smithi110.stdout:1394: writing smithi11075562-8 from 2211840 to 2449408 tid 1 2024-09-14T03:42:17.734 INFO:tasks.rados.rados.0.smithi110.stdout:1395: rollback oid 31 current snap is 143 2024-09-14T03:42:17.734 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 141 2024-09-14T03:42:17.734 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 143 2024-09-14T03:42:17.734 INFO:tasks.rados.rados.0.smithi110.stdout:1396: seq_num 445 ranges {98304=253952} 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1396: writing smithi11075562-20 from 98304 to 352256 tid 1 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1397: read oid 41 snap -1 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1397: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1398: rmattr oid 46 current snap is 143 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1398: done (4 left) 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1399: rmattr oid 32 current snap is 143 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1399: done (4 left) 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1400: snap_create 2024-09-14T03:42:17.737 INFO:tasks.rados.rados.0.smithi110.stdout:1397: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:42:17.742 INFO:tasks.rados.rados.0.smithi110.stdout:1395: finishing rollback tid 0 to smithi11075562-31 2024-09-14T03:42:17.742 INFO:tasks.rados.rados.0.smithi110.stdout:1395: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:42:17.742 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 504 (ObjNum 721 snap 0 seq_num 4276657625) dirty dne 2024-09-14T03:42:17.742 INFO:tasks.rados.rados.0.smithi110.stdout:1394: finishing write tid 1 to smithi11075562-8 2024-09-14T03:42:17.742 INFO:tasks.rados.rados.0.smithi110.stdout:1394: finishing write tid 2 to smithi11075562-8 2024-09-14T03:42:17.742 INFO:tasks.rados.rados.0.smithi110.stdout:1396: finishing write tid 1 to smithi11075562-20 2024-09-14T03:42:17.742 INFO:tasks.rados.rados.0.smithi110.stdout:1396: finishing write tid 2 to smithi11075562-20 2024-09-14T03:42:17.742 INFO:tasks.rados.rados.0.smithi110.stdout:1394: finishing write tid 3 to smithi11075562-8 2024-09-14T03:42:17.743 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 844 (ObjNum 444 snap 143 seq_num 444) dirty exists 2024-09-14T03:42:17.743 INFO:tasks.rados.rados.0.smithi110.stdout:1394: left oid 8 (ObjNum 444 snap 143 seq_num 444) 2024-09-14T03:42:17.743 INFO:tasks.rados.rados.0.smithi110.stdout:1396: finishing write tid 3 to smithi11075562-20 2024-09-14T03:42:17.743 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 569 (ObjNum 445 snap 143 seq_num 445) dirty exists 2024-09-14T03:42:17.743 INFO:tasks.rados.rados.0.smithi110.stdout:1396: left oid 20 (ObjNum 445 snap 143 seq_num 445) 2024-09-14T03:42:18.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:17 smithi110 ceph-mon[88262]: osdmap e341: 8 total, 8 up, 8 in 2024-09-14T03:42:18.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:17 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:18.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:17 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:42:18.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:17 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:18.189 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:17 smithi110 ceph-mon[86423]: osdmap e341: 8 total, 8 up, 8 in 2024-09-14T03:42:18.189 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:17 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:18.189 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:17 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:42:18.189 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:17 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:17 smithi195 ceph-mon[82388]: osdmap e341: 8 total, 8 up, 8 in 2024-09-14T03:42:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:17 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:17 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:42:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:17 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:18.682 INFO:tasks.rados.rados.0.smithi110.stdout:1394: done (4 left) 2024-09-14T03:42:18.683 INFO:tasks.rados.rados.0.smithi110.stdout:1395: done (3 left) 2024-09-14T03:42:18.683 INFO:tasks.rados.rados.0.smithi110.stdout:1396: done (2 left) 2024-09-14T03:42:18.683 INFO:tasks.rados.rados.0.smithi110.stdout:1397: done (1 left) 2024-09-14T03:42:18.683 INFO:tasks.rados.rados.0.smithi110.stdout:1400: done (0 left) 2024-09-14T03:42:18.683 INFO:tasks.rados.rados.0.smithi110.stdout:1401: rollback oid 39 current snap is 144 2024-09-14T03:42:18.683 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 139 2024-09-14T03:42:18.683 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 144 2024-09-14T03:42:18.683 INFO:tasks.rados.rados.0.smithi110.stdout:1402: seq_num 446 ranges {6168576=278528} 2024-09-14T03:42:18.718 INFO:tasks.rados.rados.0.smithi110.stdout:1402: writing smithi11075562-44 from 6168576 to 6447104 tid 1 2024-09-14T03:42:18.718 INFO:tasks.rados.rados.0.smithi110.stdout:1403: read oid 16 snap 126 2024-09-14T03:42:18.718 INFO:tasks.rados.rados.0.smithi110.stdout:1403: expect deleted 2024-09-14T03:42:18.718 INFO:tasks.rados.rados.0.smithi110.stdout:1404: read oid 17 snap -1 2024-09-14T03:42:18.718 INFO:tasks.rados.rados.0.smithi110.stdout:1404: expect deleted 2024-09-14T03:42:18.718 INFO:tasks.rados.rados.0.smithi110.stdout:1405: copy_from oid 50 from oid 25 current snap is 144 2024-09-14T03:42:18.718 INFO:tasks.rados.rados.0.smithi110.stdout:1401: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:42:18.718 INFO:tasks.rados.rados.0.smithi110.stdout:1401: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:42:18.719 INFO:tasks.rados.rados.0.smithi110.stdout:1401: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:42:18.719 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 815 (ObjNum 389 snap 129 seq_num 389) dirty exists 2024-09-14T03:42:18.719 INFO:tasks.rados.rados.0.smithi110.stdout:1401: done (4 left) 2024-09-14T03:42:18.719 INFO:tasks.rados.rados.0.smithi110.stdout:1406: rollback oid 36 current snap is 144 2024-09-14T03:42:18.719 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 36 to 107 2024-09-14T03:42:18.719 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 144 2024-09-14T03:42:18.719 INFO:tasks.rados.rados.0.smithi110.stdout:1407: seq_num 447 ranges {581632=294912} 2024-09-14T03:42:18.723 INFO:tasks.rados.rados.0.smithi110.stdout:1407: writing smithi11075562-26 from 581632 to 876544 tid 1 2024-09-14T03:42:18.723 INFO:tasks.rados.rados.0.smithi110.stdout:1408: read oid 22 snap -1 2024-09-14T03:42:18.723 INFO:tasks.rados.rados.0.smithi110.stdout:1408: expect (ObjNum 430 snap 138 seq_num 430) 2024-09-14T03:42:18.723 INFO:tasks.rados.rados.0.smithi110.stdout:1409: rmattr oid 45 current snap is 144 2024-09-14T03:42:18.723 INFO:tasks.rados.rados.0.smithi110.stdout:1409: done (7 left) 2024-09-14T03:42:18.723 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 144 2024-09-14T03:42:18.723 INFO:tasks.rados.rados.0.smithi110.stdout:1410: seq_num 448 ranges {12058624=311296} 2024-09-14T03:42:18.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[88262]: Upgrade: Updating osd.1 2024-09-14T03:42:18.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[88262]: Deploying daemon osd.1 on smithi110 2024-09-14T03:42:18.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[88262]: osdmap e342: 8 total, 8 up, 8 in 2024-09-14T03:42:18.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[88262]: pgmap v139: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 186 KiB/s wr, 10 op/s 2024-09-14T03:42:18.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[88262]: osdmap e343: 8 total, 8 up, 8 in 2024-09-14T03:42:18.785 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[86423]: Upgrade: Updating osd.1 2024-09-14T03:42:18.785 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[86423]: Deploying daemon osd.1 on smithi110 2024-09-14T03:42:18.785 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[86423]: osdmap e342: 8 total, 8 up, 8 in 2024-09-14T03:42:18.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[86423]: pgmap v139: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 186 KiB/s wr, 10 op/s 2024-09-14T03:42:18.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:18 smithi110 ceph-mon[86423]: osdmap e343: 8 total, 8 up, 8 in 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1410: writing smithi11075562-23 from 12058624 to 12369920 tid 1 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1411: copy_from oid 37 from oid 47 current snap is 144 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1402: finishing write tid 1 to smithi11075562-44 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1402: finishing write tid 2 to smithi11075562-44 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1403: done (8 left) 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1404: done (7 left) 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1412: read oid 5 snap -1 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1412: expect deleted 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 144 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:1402: finishing write tid 3 to smithi11075562-44 2024-09-14T03:42:18.792 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 388 (ObjNum 446 snap 144 seq_num 446) dirty exists 2024-09-14T03:42:18.793 INFO:tasks.rados.rados.0.smithi110.stdout:1402: left oid 44 (ObjNum 446 snap 144 seq_num 446) 2024-09-14T03:42:18.793 INFO:tasks.rados.rados.0.smithi110.stdout:1408: expect (ObjNum 430 snap 138 seq_num 430) 2024-09-14T03:42:18.820 INFO:tasks.rados.rados.0.smithi110.stdout:1406: finishing rollback tid 1 to smithi11075562-36 2024-09-14T03:42:18.820 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 859 (ObjNum 33 snap 0 seq_num 1920035672) dirty dne 2024-09-14T03:42:18.820 INFO:tasks.rados.rados.0.smithi110.stdout:1407: finishing write tid 1 to smithi11075562-26 2024-09-14T03:42:18.820 INFO:tasks.rados.rados.0.smithi110.stdout:1407: finishing write tid 2 to smithi11075562-26 2024-09-14T03:42:18.820 INFO:tasks.rados.rados.0.smithi110.stdout:1413: seq_num 449 ranges {3293184=327680} 2024-09-14T03:42:18.840 INFO:tasks.rados.rados.0.smithi110.stdout:1413: writing smithi11075562-32 from 3293184 to 3620864 tid 1 2024-09-14T03:42:18.840 INFO:tasks.rados.rados.0.smithi110.stdout:1402: done (8 left) 2024-09-14T03:42:18.840 INFO:tasks.rados.rados.0.smithi110.stdout:1406: done (7 left) 2024-09-14T03:42:18.840 INFO:tasks.rados.rados.0.smithi110.stdout:1408: done (6 left) 2024-09-14T03:42:18.840 INFO:tasks.rados.rados.0.smithi110.stdout:1414: snap_create 2024-09-14T03:42:18.840 INFO:tasks.rados.rados.0.smithi110.stdout:1407: finishing write tid 3 to smithi11075562-26 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 757 (ObjNum 447 snap 144 seq_num 447) dirty exists 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1407: left oid 26 (ObjNum 447 snap 144 seq_num 447) 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1405: finishing copy_from racing read to smithi11075562-50 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1405: finishing copy_from to smithi11075562-50 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 929 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1410: finishing write tid 1 to smithi11075562-23 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1410: finishing write tid 2 to smithi11075562-23 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1410: finishing write tid 3 to smithi11075562-23 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 515 (ObjNum 448 snap 144 seq_num 448) dirty exists 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1410: left oid 23 (ObjNum 448 snap 144 seq_num 448) 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1411: finishing copy_from racing read to smithi11075562-37 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:1411: finishing copy_from to smithi11075562-37 2024-09-14T03:42:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 947 (ObjNum 305 snap 98 seq_num 305) dirty exists 2024-09-14T03:42:18.842 INFO:tasks.rados.rados.0.smithi110.stdout:1413: finishing write tid 1 to smithi11075562-32 2024-09-14T03:42:18.842 INFO:tasks.rados.rados.0.smithi110.stdout:1413: finishing write tid 2 to smithi11075562-32 2024-09-14T03:42:18.844 INFO:tasks.rados.rados.0.smithi110.stdout:1413: finishing write tid 3 to smithi11075562-32 2024-09-14T03:42:18.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 847 (ObjNum 449 snap 144 seq_num 449) dirty exists 2024-09-14T03:42:18.844 INFO:tasks.rados.rados.0.smithi110.stdout:1413: left oid 32 (ObjNum 449 snap 144 seq_num 449) 2024-09-14T03:42:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:18 smithi195 ceph-mon[82388]: Upgrade: Updating osd.1 2024-09-14T03:42:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:18 smithi195 ceph-mon[82388]: Deploying daemon osd.1 on smithi110 2024-09-14T03:42:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:18 smithi195 ceph-mon[82388]: osdmap e342: 8 total, 8 up, 8 in 2024-09-14T03:42:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:18 smithi195 ceph-mon[82388]: pgmap v139: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 186 KiB/s wr, 10 op/s 2024-09-14T03:42:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:18 smithi195 ceph-mon[82388]: osdmap e343: 8 total, 8 up, 8 in 2024-09-14T03:42:19.733 INFO:tasks.rados.rados.0.smithi110.stdout:1405: done (6 left) 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:1407: done (5 left) 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:1410: done (4 left) 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:1411: done (3 left) 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:1412: done (2 left) 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:1413: done (1 left) 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:1414: done (0 left) 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:1415: copy_from oid 49 from oid 24 current snap is 145 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:1416: rollback oid 15 current snap is 145 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 139 2024-09-14T03:42:19.734 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 145 2024-09-14T03:42:19.735 INFO:tasks.rados.rados.0.smithi110.stdout:1417: seq_num 450 ranges {0=344064} 2024-09-14T03:42:19.735 INFO:tasks.rados.rados.0.smithi110.stdout:1417: writing smithi11075562-35 from 0 to 344064 tid 1 2024-09-14T03:42:19.735 INFO:tasks.rados.rados.0.smithi110.stdout:1418: snap_create 2024-09-14T03:42:19.737 INFO:tasks.rados.rados.0.smithi110.stdout:1416: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:42:19.737 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 669 (ObjNum 177 snap 0 seq_num 2917113216) dirty dne 2024-09-14T03:42:19.739 INFO:tasks.rados.rados.0.smithi110.stdout:1417: finishing write tid 1 to smithi11075562-35 2024-09-14T03:42:19.739 INFO:tasks.rados.rados.0.smithi110.stdout:1417: finishing write tid 2 to smithi11075562-35 2024-09-14T03:42:19.740 INFO:tasks.rados.rados.0.smithi110.stdout:1417: finishing write tid 3 to smithi11075562-35 2024-09-14T03:42:19.740 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 854 (ObjNum 450 snap 145 seq_num 450) dirty exists 2024-09-14T03:42:19.740 INFO:tasks.rados.rados.0.smithi110.stdout:1417: left oid 35 (ObjNum 450 snap 145 seq_num 450) 2024-09-14T03:42:19.763 INFO:tasks.rados.rados.0.smithi110.stdout:1415: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:42:19.766 INFO:tasks.rados.rados.0.smithi110.stdout:1415: finishing copy_from to smithi11075562-49 2024-09-14T03:42:19.766 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 512 (ObjNum 432 snap 138 seq_num 432) dirty exists 2024-09-14T03:42:20.028 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:19 smithi110 systemd[1]: Stopping Ceph osd.1 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:42:20.446 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[45099]: 2024-09-14T03:42:20.033+0000 7f63477bd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:42:20.446 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[45099]: 2024-09-14T03:42:20.033+0000 7f63477bd640 -1 osd.1 343 *** Got signal Terminated *** 2024-09-14T03:42:20.446 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[45099]: 2024-09-14T03:42:20.033+0000 7f63477bd640 -1 osd.1 343 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:42:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:20.744 INFO:tasks.rados.rados.0.smithi110.stdout:1415: done (3 left) 2024-09-14T03:42:20.744 INFO:tasks.rados.rados.0.smithi110.stdout:1416: done (2 left) 2024-09-14T03:42:20.744 INFO:tasks.rados.rados.0.smithi110.stdout:1417: done (1 left) 2024-09-14T03:42:20.744 INFO:tasks.rados.rados.0.smithi110.stdout:1418: done (0 left) 2024-09-14T03:42:20.744 INFO:tasks.rados.rados.0.smithi110.stdout:1419: rollback oid 1 current snap is 146 2024-09-14T03:42:20.745 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 1 to 139 2024-09-14T03:42:20.745 INFO:tasks.rados.rados.0.smithi110.stdout:1420: rollback oid 36 current snap is 146 2024-09-14T03:42:20.745 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 36 to 126 2024-09-14T03:42:20.745 INFO:tasks.rados.rados.0.smithi110.stdout:1421: snap_remove snap 121 2024-09-14T03:42:20.895 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:42:21.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-mon[86423]: pgmap v142: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 264 KiB/s wr, 11 op/s 2024-09-14T03:42:21.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-mon[86423]: osdmap e344: 8 total, 8 up, 8 in 2024-09-14T03:42:21.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-mon[86423]: osd.1 marked itself down and dead 2024-09-14T03:42:21.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-mon[88262]: pgmap v142: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 264 KiB/s wr, 11 op/s 2024-09-14T03:42:21.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-mon[88262]: osdmap e344: 8 total, 8 up, 8 in 2024-09-14T03:42:21.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:20 smithi110 ceph-mon[88262]: osd.1 marked itself down and dead 2024-09-14T03:42:21.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:20 smithi195 ceph-mon[82388]: pgmap v142: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 264 KiB/s wr, 11 op/s 2024-09-14T03:42:21.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:20 smithi195 ceph-mon[82388]: osdmap e344: 8 total, 8 up, 8 in 2024-09-14T03:42:21.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:20 smithi195 ceph-mon[82388]: osd.1 marked itself down and dead 2024-09-14T03:42:21.464 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:42:21.464 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (3m) 24s ago 7m 16.6M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:42:21.464 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (2m) 63s ago 7m 78.7M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:42:21.464 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (8m) 24s ago 8m 19.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:42:21.464 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (8m) 63s ago 8m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:42:21.464 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (2m) 63s ago 11m 452M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:42:21.464 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (2m) 24s ago 13m 523M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (108s) 24s ago 13m 48.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (68s) 63s ago 12m 21.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (85s) 24s ago 12m 40.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (2m) 24s ago 6m 9680k - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (2m) 63s ago 6m 9722k - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (28s) 24s ago 11m 11.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (11m) 24s ago 11m 185M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e44a53ff5879 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (10m) 24s ago 10m 154M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (10m) 24s ago 10m 134M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (10m) 63s ago 10m 229M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (9m) 63s ago 9m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (9m) 63s ago 9m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (8m) 63s ago 8m 106M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:42:21.465 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (2m) 63s ago 7m 44.2M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:42:21.562 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:21 smithi110 podman[97356]: 2024-09-14 03:42:21.306242357 +0000 UTC m=+1.386335281 container died e44a53ff587942dda934a8bf7400f6f0e1cefbd0c493f82ac9bbecbdf75a19b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:42:21.741 INFO:tasks.rados.rados.0.smithi110.stdout:1421: done (2 left) 2024-09-14T03:42:21.741 INFO:tasks.rados.rados.0.smithi110.stdout:1422: snap_remove snap 138 2024-09-14T03:42:21.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:21 smithi110 ceph-mon[88262]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:42:21.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:21 smithi110 ceph-mon[88262]: osdmap e345: 8 total, 7 up, 8 in 2024-09-14T03:42:21.818 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:42:21.818 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:42:21.818 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:42:21.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:42:21.818 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:42:21.818 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:42:21.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 6, 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8, 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 6 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:42:21.819 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:42:21.820 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:42:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:42:21] "GET /metrics HTTP/1.1" 200 36250 "" "Prometheus/2.51.0" 2024-09-14T03:42:21.820 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:21 smithi110 ceph-mon[86423]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:42:21.820 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:21 smithi110 ceph-mon[86423]: osdmap e345: 8 total, 7 up, 8 in 2024-09-14T03:42:22.071 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:21 smithi110 podman[97356]: 2024-09-14 03:42:21.877386776 +0000 UTC m=+1.957479697 container cleanup e44a53ff587942dda934a8bf7400f6f0e1cefbd0c493f82ac9bbecbdf75a19b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, org.label-schema.build-date=20240731) 2024-09-14T03:42:22.071 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:21 smithi110 bash[97356]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:42:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "6/20 daemons upgraded", 2024-09-14T03:42:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:42:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:42:22.116 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:42:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:21 smithi195 ceph-mon[82388]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:42:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:21 smithi195 ceph-mon[82388]: osdmap e345: 8 total, 7 up, 8 in 2024-09-14T03:42:22.446 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:22 smithi110 podman[97451]: 2024-09-14 03:42:22.070594341 +0000 UTC m=+0.757940360 container remove e44a53ff587942dda934a8bf7400f6f0e1cefbd0c493f82ac9bbecbdf75a19b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:42:22.741 INFO:tasks.rados.rados.0.smithi110.stdout:1422: done (2 left) 2024-09-14T03:42:22.745 INFO:tasks.rados.rados.0.smithi110.stdout:1423: snap_remove snap 110 2024-09-14T03:42:22.746 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:22 smithi110 podman[97585]: 2024-09-14 03:42:22.391668307 +0000 UTC m=+0.015858577 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:42:22.746 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:22 smithi110 podman[97585]: 2024-09-14 03:42:22.505172345 +0000 UTC m=+0.129362606 container create 16e06e60bba983e0da02c6274b808ba0c811aa0429d41d0cdc1e67454e4819b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-a278231, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-14T03:42:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[86423]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[86423]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[86423]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[86423]: pgmap v144: 121 pgs: 1 stale+active+clean+snaptrim, 15 stale+active+clean, 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 98 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 263 KiB/s wr, 11 op/s 2024-09-14T03:42:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[86423]: osdmap e346: 8 total, 7 up, 8 in 2024-09-14T03:42:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1162352706' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[86423]: osdmap e347: 8 total, 7 up, 8 in 2024-09-14T03:42:23.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[88262]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[88262]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[88262]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[88262]: pgmap v144: 121 pgs: 1 stale+active+clean+snaptrim, 15 stale+active+clean, 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 98 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 263 KiB/s wr, 11 op/s 2024-09-14T03:42:23.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[88262]: osdmap e346: 8 total, 7 up, 8 in 2024-09-14T03:42:23.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1162352706' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:23.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:22 smithi110 ceph-mon[88262]: osdmap e347: 8 total, 7 up, 8 in 2024-09-14T03:42:23.198 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:22 smithi110 podman[97585]: 2024-09-14 03:42:22.741515958 +0000 UTC m=+0.365706234 container init 16e06e60bba983e0da02c6274b808ba0c811aa0429d41d0cdc1e67454e4819b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T03:42:23.198 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:22 smithi110 podman[97585]: 2024-09-14 03:42:22.747018922 +0000 UTC m=+0.371209187 container start 16e06e60bba983e0da02c6274b808ba0c811aa0429d41d0cdc1e67454e4819b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:42:23.198 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:22 smithi110 podman[97585]: 2024-09-14 03:42:22.841640768 +0000 UTC m=+0.465831039 container attach 16e06e60bba983e0da02c6274b808ba0c811aa0429d41d0cdc1e67454e4819b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:42:23.198 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:22 smithi110 podman[97585]: 2024-09-14 03:42:22.929834982 +0000 UTC m=+0.554025247 container died 16e06e60bba983e0da02c6274b808ba0c811aa0429d41d0cdc1e67454e4819b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T03:42:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:22 smithi195 ceph-mon[82388]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:22 smithi195 ceph-mon[82388]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:22 smithi195 ceph-mon[82388]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:22 smithi195 ceph-mon[82388]: pgmap v144: 121 pgs: 1 stale+active+clean+snaptrim, 15 stale+active+clean, 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 98 active+clean; 393 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 263 KiB/s wr, 11 op/s 2024-09-14T03:42:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:22 smithi195 ceph-mon[82388]: osdmap e346: 8 total, 7 up, 8 in 2024-09-14T03:42:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:22 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1162352706' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:22 smithi195 ceph-mon[82388]: osdmap e347: 8 total, 7 up, 8 in 2024-09-14T03:42:23.683 INFO:tasks.rados.rados.0.smithi110.stdout:1423: done (2 left) 2024-09-14T03:42:23.683 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 146 2024-09-14T03:42:23.683 INFO:tasks.rados.rados.0.smithi110.stdout:1419: finishing rollback tid 0 to smithi11075562-1 2024-09-14T03:42:23.684 INFO:tasks.rados.rados.0.smithi110.stdout:1424: seq_num 451 ranges {24576=360448} 2024-09-14T03:42:23.686 INFO:tasks.rados.rados.0.smithi110.stdout:1424: writing smithi11075562-29 from 24576 to 385024 tid 1 2024-09-14T03:42:23.686 INFO:tasks.rados.rados.0.smithi110.stdout:1425: snap_remove snap 131 2024-09-14T03:42:23.686 INFO:tasks.rados.rados.0.smithi110.stdout:1419: finishing rollback tid 1 to smithi11075562-1 2024-09-14T03:42:23.686 INFO:tasks.rados.rados.0.smithi110.stdout:1419: finishing rollback tid 2 to smithi11075562-1 2024-09-14T03:42:23.686 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 860 (ObjNum 424 snap 136 seq_num 424) dirty exists 2024-09-14T03:42:23.686 INFO:tasks.rados.rados.0.smithi110.stdout:1420: finishing rollback tid 1 to smithi11075562-36 2024-09-14T03:42:23.686 INFO:tasks.rados.rados.0.smithi110.stdout:1420: finishing rollback tid 2 to smithi11075562-36 2024-09-14T03:42:23.686 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 957 (ObjNum 372 snap 122 seq_num 372) dirty exists 2024-09-14T03:42:23.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:23 smithi110 ceph-mon[88262]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:23 smithi110 ceph-mon[88262]: osdmap e348: 8 total, 7 up, 8 in 2024-09-14T03:42:23.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:23 smithi110 ceph-mon[86423]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:23.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:23 smithi110 ceph-mon[86423]: osdmap e348: 8 total, 7 up, 8 in 2024-09-14T03:42:23.947 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:23 smithi110 podman[97608]: 2024-09-14 03:42:23.504116665 +0000 UTC m=+0.568739196 container remove 16e06e60bba983e0da02c6274b808ba0c811aa0429d41d0cdc1e67454e4819b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-14T03:42:23.947 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:23 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-14T03:42:23.947 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:23 smithi110 systemd[1]: Stopped Ceph osd.1 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:42:23.947 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:23 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.1.service: Consumed 10.974s CPU time. 2024-09-14T03:42:23.947 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:23 smithi110 systemd[1]: Starting Ceph osd.1 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:42:24.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:23 smithi195 ceph-mon[82388]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:24.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:23 smithi195 ceph-mon[82388]: osdmap e348: 8 total, 7 up, 8 in 2024-09-14T03:42:24.267 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:23 smithi110 podman[97708]: 2024-09-14 03:42:23.862499319 +0000 UTC m=+0.015372126 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:42:24.267 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:23 smithi110 podman[97708]: 2024-09-14 03:42:23.980423169 +0000 UTC m=+0.133295964 container create 44a3027fd3d5b950b3b6c2ad414087f34d18681cd693b5b37be893eadc8a786f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-14T03:42:24.267 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 podman[97708]: 2024-09-14 03:42:24.191384464 +0000 UTC m=+0.344257258 container init 44a3027fd3d5b950b3b6c2ad414087f34d18681cd693b5b37be893eadc8a786f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T03:42:24.267 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 podman[97708]: 2024-09-14 03:42:24.194781952 +0000 UTC m=+0.347654753 container start 44a3027fd3d5b950b3b6c2ad414087f34d18681cd693b5b37be893eadc8a786f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:42:24.267 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 podman[97708]: 2024-09-14 03:42:24.266301117 +0000 UTC m=+0.419173926 container attach 44a3027fd3d5b950b3b6c2ad414087f34d18681cd693b5b37be893eadc8a786f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True) 2024-09-14T03:42:24.458 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:24 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:24.687 INFO:tasks.rados.rados.0.smithi110.stdout:1419: done (3 left) 2024-09-14T03:42:24.687 INFO:tasks.rados.rados.0.smithi110.stdout:1420: done (2 left) 2024-09-14T03:42:24.687 INFO:tasks.rados.rados.0.smithi110.stdout:1425: done (1 left) 2024-09-14T03:42:24.687 INFO:tasks.rados.rados.0.smithi110.stdout:1426: delete oid 30 current snap is 146 2024-09-14T03:42:24.687 INFO:tasks.rados.rados.0.smithi110.stdout:1424: finishing write tid 1 to smithi11075562-29 2024-09-14T03:42:24.688 INFO:tasks.rados.rados.0.smithi110.stdout:1424: finishing write tid 2 to smithi11075562-29 2024-09-14T03:42:24.688 INFO:tasks.rados.rados.0.smithi110.stdout:1424: finishing write tid 3 to smithi11075562-29 2024-09-14T03:42:24.688 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 482 (ObjNum 451 snap 146 seq_num 451) dirty exists 2024-09-14T03:42:24.688 INFO:tasks.rados.rados.0.smithi110.stdout:1424: left oid 29 (ObjNum 451 snap 146 seq_num 451) 2024-09-14T03:42:24.693 INFO:tasks.rados.rados.0.smithi110.stdout:1424: done (1 left) 2024-09-14T03:42:24.693 INFO:tasks.rados.rados.0.smithi110.stdout:1426: done (0 left) 2024-09-14T03:42:24.693 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 146 2024-09-14T03:42:24.694 INFO:tasks.rados.rados.0.smithi110.stdout:1427: seq_num 453 ranges {0=393216} 2024-09-14T03:42:24.696 INFO:tasks.rados.rados.0.smithi110.stdout:1427: writing smithi11075562-15 from 0 to 393216 tid 1 2024-09-14T03:42:24.696 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:42:24.696 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 bash[97708]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:42:24.696 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:42:24.696 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 bash[97708]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:42:24.697 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 146 2024-09-14T03:42:24.697 INFO:tasks.rados.rados.0.smithi110.stdout:1428: seq_num 454 ranges {589824=409600} 2024-09-14T03:42:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1428: writing smithi11075562-22 from 589824 to 999424 tid 1 2024-09-14T03:42:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1429: read oid 30 snap -1 2024-09-14T03:42:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1429: expect deleted 2024-09-14T03:42:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1430: read oid 50 snap -1 2024-09-14T03:42:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1430: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:42:24.704 INFO:tasks.rados.rados.0.smithi110.stdout:1431: snap_create 2024-09-14T03:42:24.704 INFO:tasks.rados.rados.0.smithi110.stdout:1427: finishing write tid 1 to smithi11075562-15 2024-09-14T03:42:24.704 INFO:tasks.rados.rados.0.smithi110.stdout:1427: finishing write tid 2 to smithi11075562-15 2024-09-14T03:42:24.706 INFO:tasks.rados.rados.0.smithi110.stdout:1427: finishing write tid 3 to smithi11075562-15 2024-09-14T03:42:24.706 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 881 (ObjNum 453 snap 146 seq_num 453) dirty exists 2024-09-14T03:42:24.707 INFO:tasks.rados.rados.0.smithi110.stdout:1427: left oid 15 (ObjNum 453 snap 146 seq_num 453) 2024-09-14T03:42:24.708 INFO:tasks.rados.rados.0.smithi110.stdout:1428: finishing write tid 1 to smithi11075562-22 2024-09-14T03:42:24.714 INFO:tasks.rados.rados.0.smithi110.stdout:1428: finishing write tid 2 to smithi11075562-22 2024-09-14T03:42:24.714 INFO:tasks.rados.rados.0.smithi110.stdout:1428: finishing write tid 3 to smithi11075562-22 2024-09-14T03:42:24.715 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 884 (ObjNum 454 snap 146 seq_num 454) dirty exists 2024-09-14T03:42:24.715 INFO:tasks.rados.rados.0.smithi110.stdout:1428: left oid 22 (ObjNum 454 snap 146 seq_num 454) 2024-09-14T03:42:24.722 INFO:tasks.rados.rados.0.smithi110.stdout:1430: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:42:25.084 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-mon[88262]: pgmap v148: 121 pgs: 26 active+undersized, 2 stale+active+clean, 10 active+undersized+degraded, 83 active+clean; 411 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 459 KiB/s rd, 3.9 MiB/s wr, 10 op/s; 79/834 objects degraded (9.472%) 2024-09-14T03:42:25.084 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-mon[88262]: Health check failed: Degraded data redundancy: 79/834 objects degraded (9.472%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:25.085 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-mon[88262]: osdmap e349: 8 total, 7 up, 8 in 2024-09-14T03:42:25.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-mon[86423]: pgmap v148: 121 pgs: 26 active+undersized, 2 stale+active+clean, 10 active+undersized+degraded, 83 active+clean; 411 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 459 KiB/s rd, 3.9 MiB/s wr, 10 op/s; 79/834 objects degraded (9.472%) 2024-09-14T03:42:25.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-mon[86423]: Health check failed: Degraded data redundancy: 79/834 objects degraded (9.472%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:25.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-mon[86423]: osdmap e349: 8 total, 7 up, 8 in 2024-09-14T03:42:25.085 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:42:25.085 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 bash[97708]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:42:25.086 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:42:25.086 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:24 smithi110 bash[97708]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:42:25.086 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:42:25.086 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 bash[97708]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:42:25.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:24 smithi195 ceph-mon[82388]: pgmap v148: 121 pgs: 26 active+undersized, 2 stale+active+clean, 10 active+undersized+degraded, 83 active+clean; 411 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 459 KiB/s rd, 3.9 MiB/s wr, 10 op/s; 79/834 objects degraded (9.472%) 2024-09-14T03:42:25.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:24 smithi195 ceph-mon[82388]: Health check failed: Degraded data redundancy: 79/834 objects degraded (9.472%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:25.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:24 smithi195 ceph-mon[82388]: osdmap e349: 8 total, 7 up, 8 in 2024-09-14T03:42:25.412 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:42:25.412 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 bash[97708]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:42:25.412 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-14T03:42:25.412 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 bash[97708]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-14T03:42:25.412 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:42:25.412 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 bash[97708]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:42:25.412 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate[97724]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-14T03:42:25.413 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 bash[97708]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-14T03:42:25.413 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 podman[97708]: 2024-09-14 03:42:25.111359921 +0000 UTC m=+1.264232732 container died 44a3027fd3d5b950b3b6c2ad414087f34d18681cd693b5b37be893eadc8a786f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T03:42:25.690 INFO:tasks.rados.rados.0.smithi110.stdout:1427: done (4 left) 2024-09-14T03:42:25.691 INFO:tasks.rados.rados.0.smithi110.stdout:1428: done (3 left) 2024-09-14T03:42:25.691 INFO:tasks.rados.rados.0.smithi110.stdout:1429: done (2 left) 2024-09-14T03:42:25.691 INFO:tasks.rados.rados.0.smithi110.stdout:1430: done (1 left) 2024-09-14T03:42:25.691 INFO:tasks.rados.rados.0.smithi110.stdout:1431: done (0 left) 2024-09-14T03:42:25.691 INFO:tasks.rados.rados.0.smithi110.stdout:1432: rmattr oid 33 current snap is 147 2024-09-14T03:42:25.692 INFO:tasks.rados.rados.0.smithi110.stdout:1433: copy_from oid 11 from oid 22 current snap is 147 2024-09-14T03:42:25.692 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 147 2024-09-14T03:42:25.692 INFO:tasks.rados.rados.0.smithi110.stdout:1434: seq_num 455 ranges {6815744=425984} 2024-09-14T03:42:25.730 INFO:tasks.rados.rados.0.smithi110.stdout:1434: writing smithi11075562-2 from 6815744 to 7241728 tid 1 2024-09-14T03:42:25.730 INFO:tasks.rados.rados.0.smithi110.stdout:1435: copy_from oid 42 from oid 27 current snap is 147 2024-09-14T03:42:25.730 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 0 (ObjNum 129 snap 0 seq_num 64718528) dirty dne 2024-09-14T03:42:25.730 INFO:tasks.rados.rados.0.smithi110.stdout:1433: finishing copy_from racing read to smithi11075562-11 2024-09-14T03:42:25.730 INFO:tasks.rados.rados.0.smithi110.stdout:1433: finishing copy_from to smithi11075562-11 2024-09-14T03:42:25.730 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 884 (ObjNum 454 snap 146 seq_num 454) dirty exists 2024-09-14T03:42:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:1432: done (3 left) 2024-09-14T03:42:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:1433: done (2 left) 2024-09-14T03:42:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:1436: read oid 38 snap -1 2024-09-14T03:42:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:1436: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:42:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:1437: snap_create 2024-09-14T03:42:25.734 INFO:tasks.rados.rados.0.smithi110.stdout:1434: finishing write tid 1 to smithi11075562-2 2024-09-14T03:42:25.734 INFO:tasks.rados.rados.0.smithi110.stdout:1434: finishing write tid 2 to smithi11075562-2 2024-09-14T03:42:25.736 INFO:tasks.rados.rados.0.smithi110.stdout:1436: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:42:25.754 INFO:tasks.rados.rados.0.smithi110.stdout:1434: finishing write tid 3 to smithi11075562-2 2024-09-14T03:42:25.754 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 683 (ObjNum 455 snap 147 seq_num 455) dirty exists 2024-09-14T03:42:25.755 INFO:tasks.rados.rados.0.smithi110.stdout:1434: left oid 2 (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:42:25.758 INFO:tasks.rados.rados.0.smithi110.stdout:1435: finishing copy_from racing read to smithi11075562-42 2024-09-14T03:42:25.761 INFO:tasks.rados.rados.0.smithi110.stdout:1435: finishing copy_from to smithi11075562-42 2024-09-14T03:42:25.761 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 678 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:25.946 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 podman[97865]: 2024-09-14 03:42:25.670940854 +0000 UTC m=+0.554019513 container remove 44a3027fd3d5b950b3b6c2ad414087f34d18681cd693b5b37be893eadc8a786f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231) 2024-09-14T03:42:25.946 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:25 smithi110 podman[97922]: 2024-09-14 03:42:25.789643338 +0000 UTC m=+0.015478487 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:42:26.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:26.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:26.301 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:26 smithi110 podman[97922]: 2024-09-14 03:42:26.005658424 +0000 UTC m=+0.231493576 container create 6ed2d900d1e261d3b988047e95463380c2bb2e5710bc3c1cadf65256b7ff3fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, ceph=True, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-14T03:42:26.301 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:26 smithi110 podman[97922]: 2024-09-14 03:42:26.208003168 +0000 UTC m=+0.433838318 container init 6ed2d900d1e261d3b988047e95463380c2bb2e5710bc3c1cadf65256b7ff3fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-14T03:42:26.301 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:26 smithi110 podman[97922]: 2024-09-14 03:42:26.211196919 +0000 UTC m=+0.437032077 container start 6ed2d900d1e261d3b988047e95463380c2bb2e5710bc3c1cadf65256b7ff3fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-14T03:42:26.301 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-osd[97941]: -- 172.21.15.110:0/905283021 <== mon.0 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a298e6fa40 con 0x55a299c42000 2024-09-14T03:42:26.301 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:26 smithi110 bash[97922]: 6ed2d900d1e261d3b988047e95463380c2bb2e5710bc3c1cadf65256b7ff3fd6 2024-09-14T03:42:26.690 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:26 smithi110 systemd[1]: Started Ceph osd.1 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:42:26.694 INFO:tasks.rados.rados.0.smithi110.stdout:1434: done (3 left) 2024-09-14T03:42:26.694 INFO:tasks.rados.rados.0.smithi110.stdout:1435: done (2 left) 2024-09-14T03:42:26.694 INFO:tasks.rados.rados.0.smithi110.stdout:1436: done (1 left) 2024-09-14T03:42:26.694 INFO:tasks.rados.rados.0.smithi110.stdout:1437: done (0 left) 2024-09-14T03:42:26.694 INFO:tasks.rados.rados.0.smithi110.stdout:1438: snap_create 2024-09-14T03:42:26.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[88262]: osdmap e350: 8 total, 7 up, 8 in 2024-09-14T03:42:26.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[88262]: pgmap v151: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 411 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 460 KiB/s rd, 3.9 MiB/s wr, 10 op/s; 97/834 objects degraded (11.631%) 2024-09-14T03:42:26.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:26.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:26.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:26.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[86423]: osdmap e350: 8 total, 7 up, 8 in 2024-09-14T03:42:26.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[86423]: pgmap v151: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 411 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 460 KiB/s rd, 3.9 MiB/s wr, 10 op/s; 97/834 objects degraded (11.631%) 2024-09-14T03:42:26.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:26.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:26.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:26 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:26.949 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:26 smithi195 ceph-mon[82388]: osdmap e350: 8 total, 7 up, 8 in 2024-09-14T03:42:26.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:26 smithi195 ceph-mon[82388]: pgmap v151: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 411 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 460 KiB/s rd, 3.9 MiB/s wr, 10 op/s; 97/834 objects degraded (11.631%) 2024-09-14T03:42:26.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:26 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:26.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:26 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:26.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:26 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:27.208 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:26 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:27.695 INFO:tasks.rados.rados.0.smithi110.stdout:1438: done (0 left) 2024-09-14T03:42:27.695 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 149 2024-09-14T03:42:27.695 INFO:tasks.rados.rados.0.smithi110.stdout:1439: seq_num 456 ranges {0=442368} 2024-09-14T03:42:27.698 INFO:tasks.rados.rados.0.smithi110.stdout:1439: writing smithi11075562-7 from 0 to 442368 tid 1 2024-09-14T03:42:27.698 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 149 2024-09-14T03:42:27.698 INFO:tasks.rados.rados.0.smithi110.stdout:1440: seq_num 457 ranges {3588096=458752} 2024-09-14T03:42:27.720 INFO:tasks.rados.rados.0.smithi110.stdout:1440: writing smithi11075562-45 from 3588096 to 4046848 tid 1 2024-09-14T03:42:27.720 INFO:tasks.rados.rados.0.smithi110.stdout:1441: copy_from oid 38 from oid 44 current snap is 149 2024-09-14T03:42:27.720 INFO:tasks.rados.rados.0.smithi110.stdout:1439: finishing write tid 1 to smithi11075562-7 2024-09-14T03:42:27.720 INFO:tasks.rados.rados.0.smithi110.stdout:1439: finishing write tid 2 to smithi11075562-7 2024-09-14T03:42:27.721 INFO:tasks.rados.rados.0.smithi110.stdout:1439: finishing write tid 3 to smithi11075562-7 2024-09-14T03:42:27.721 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 484 (ObjNum 456 snap 149 seq_num 456) dirty exists 2024-09-14T03:42:27.721 INFO:tasks.rados.rados.0.smithi110.stdout:1439: left oid 7 (ObjNum 456 snap 149 seq_num 456) 2024-09-14T03:42:27.721 INFO:tasks.rados.rados.0.smithi110.stdout:1439: done (2 left) 2024-09-14T03:42:27.721 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 149 2024-09-14T03:42:27.721 INFO:tasks.rados.rados.0.smithi110.stdout:1442: seq_num 458 ranges {6365184=475136} 2024-09-14T03:42:27.758 INFO:tasks.rados.rados.0.smithi110.stdout:1442: writing smithi11075562-27 from 6365184 to 6840320 tid 1 2024-09-14T03:42:27.758 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 149 2024-09-14T03:42:27.758 INFO:tasks.rados.rados.0.smithi110.stdout:1440: finishing write tid 1 to smithi11075562-45 2024-09-14T03:42:27.758 INFO:tasks.rados.rados.0.smithi110.stdout:1443: seq_num 459 ranges {0=491520} 2024-09-14T03:42:27.761 INFO:tasks.rados.rados.0.smithi110.stdout:1443: writing smithi11075562-5 from 0 to 491520 tid 1 2024-09-14T03:42:27.761 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 149 2024-09-14T03:42:27.761 INFO:tasks.rados.rados.0.smithi110.stdout:1440: finishing write tid 2 to smithi11075562-45 2024-09-14T03:42:27.761 INFO:tasks.rados.rados.0.smithi110.stdout:1440: finishing write tid 3 to smithi11075562-45 2024-09-14T03:42:27.762 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 988 (ObjNum 457 snap 149 seq_num 457) dirty exists 2024-09-14T03:42:27.762 INFO:tasks.rados.rados.0.smithi110.stdout:1440: left oid 45 (ObjNum 457 snap 149 seq_num 457) 2024-09-14T03:42:27.762 INFO:tasks.rados.rados.0.smithi110.stdout:1441: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:42:27.762 INFO:tasks.rados.rados.0.smithi110.stdout:1441: finishing copy_from to smithi11075562-38 2024-09-14T03:42:27.762 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 388 (ObjNum 446 snap 144 seq_num 446) dirty exists 2024-09-14T03:42:27.762 INFO:tasks.rados.rados.0.smithi110.stdout:1444: seq_num 460 ranges {1556480=507904} 2024-09-14T03:42:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:1444: writing smithi11075562-37 from 1556480 to 2064384 tid 1 2024-09-14T03:42:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:1440: done (4 left) 2024-09-14T03:42:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:1441: done (3 left) 2024-09-14T03:42:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:1445: snap_remove snap 148 2024-09-14T03:42:27.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:27 smithi110 ceph-mon[88262]: osdmap e351: 8 total, 7 up, 8 in 2024-09-14T03:42:27.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:27 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:27.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:27 smithi110 ceph-mon[86423]: osdmap e351: 8 total, 7 up, 8 in 2024-09-14T03:42:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:27 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:27.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:27 smithi195 ceph-mon[82388]: osdmap e351: 8 total, 7 up, 8 in 2024-09-14T03:42:27.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:27 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:28.446 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[97937]: 2024-09-14T03:42:28.081+0000 7f24ad37c640 -1 Falling back to public interface 2024-09-14T03:42:28.685 INFO:tasks.rados.rados.0.smithi110.stdout:1445: done (3 left) 2024-09-14T03:42:28.685 INFO:tasks.rados.rados.0.smithi110.stdout:1446: snap_create 2024-09-14T03:42:28.685 INFO:tasks.rados.rados.0.smithi110.stdout:1443: finishing write tid 1 to smithi11075562-5 2024-09-14T03:42:28.685 INFO:tasks.rados.rados.0.smithi110.stdout:1443: finishing write tid 2 to smithi11075562-5 2024-09-14T03:42:28.685 INFO:tasks.rados.rados.0.smithi110.stdout:1442: finishing write tid 1 to smithi11075562-27 2024-09-14T03:42:28.685 INFO:tasks.rados.rados.0.smithi110.stdout:1442: finishing write tid 2 to smithi11075562-27 2024-09-14T03:42:28.685 INFO:tasks.rados.rados.0.smithi110.stdout:1442: finishing write tid 3 to smithi11075562-27 2024-09-14T03:42:28.685 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 680 (ObjNum 458 snap 149 seq_num 458) dirty exists 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:1442: left oid 27 (ObjNum 458 snap 149 seq_num 458) 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:1443: finishing write tid 3 to smithi11075562-5 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 990 (ObjNum 459 snap 149 seq_num 459) dirty exists 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:1443: left oid 5 (ObjNum 459 snap 149 seq_num 459) 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:1444: finishing write tid 1 to smithi11075562-37 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:1444: finishing write tid 2 to smithi11075562-37 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:1444: finishing write tid 3 to smithi11075562-37 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 993 (ObjNum 460 snap 149 seq_num 460) dirty exists 2024-09-14T03:42:28.686 INFO:tasks.rados.rados.0.smithi110.stdout:1444: left oid 37 (ObjNum 460 snap 149 seq_num 460) 2024-09-14T03:42:28.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:28 smithi110 ceph-mon[88262]: osdmap e352: 8 total, 7 up, 8 in 2024-09-14T03:42:28.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:28 smithi110 ceph-mon[88262]: pgmap v154: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 419 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.1 MiB/s wr, 5 op/s; 98/843 objects degraded (11.625%) 2024-09-14T03:42:28.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:28 smithi110 ceph-mon[88262]: osdmap e353: 8 total, 7 up, 8 in 2024-09-14T03:42:28.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:28 smithi110 ceph-mon[86423]: osdmap e352: 8 total, 7 up, 8 in 2024-09-14T03:42:28.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:28 smithi110 ceph-mon[86423]: pgmap v154: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 419 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.1 MiB/s wr, 5 op/s; 98/843 objects degraded (11.625%) 2024-09-14T03:42:28.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:28 smithi110 ceph-mon[86423]: osdmap e353: 8 total, 7 up, 8 in 2024-09-14T03:42:28.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:28 smithi195 ceph-mon[82388]: osdmap e352: 8 total, 7 up, 8 in 2024-09-14T03:42:28.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:28 smithi195 ceph-mon[82388]: pgmap v154: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 419 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.1 MiB/s wr, 5 op/s; 98/843 objects degraded (11.625%) 2024-09-14T03:42:28.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:28 smithi195 ceph-mon[82388]: osdmap e353: 8 total, 7 up, 8 in 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:1442: done (3 left) 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:1443: done (2 left) 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:1444: done (1 left) 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:1446: done (0 left) 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:1447: copy_from oid 34 from oid 38 current snap is 150 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:1448: rollback oid 46 current snap is 150 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 46 to 107 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:1449: rollback oid 23 current snap is 150 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 23 to 129 2024-09-14T03:42:29.687 INFO:tasks.rados.rados.0.smithi110.stdout:1450: delete oid 22 current snap is 150 2024-09-14T03:42:29.692 INFO:tasks.rados.rados.0.smithi110.stdout:1450: done (3 left) 2024-09-14T03:42:29.693 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 150 2024-09-14T03:42:29.693 INFO:tasks.rados.rados.0.smithi110.stdout:1451: seq_num 462 ranges {3817472=540672} 2024-09-14T03:42:29.718 INFO:tasks.rados.rados.0.smithi110.stdout:1451: writing smithi11075562-12 from 3817472 to 4358144 tid 1 2024-09-14T03:42:29.718 INFO:tasks.rados.rados.0.smithi110.stdout:1452: copy_from oid 28 from oid 42 current snap is 150 2024-09-14T03:42:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1448: finishing rollback tid 0 to smithi11075562-46 2024-09-14T03:42:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1448: finishing rollback tid 1 to smithi11075562-46 2024-09-14T03:42:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1448: finishing rollback tid 2 to smithi11075562-46 2024-09-14T03:42:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 573 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:42:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1449: finishing rollback tid 0 to smithi11075562-23 2024-09-14T03:42:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1449: finishing rollback tid 1 to smithi11075562-23 2024-09-14T03:42:29.719 INFO:tasks.rados.rados.0.smithi110.stdout:1449: finishing rollback tid 2 to smithi11075562-23 2024-09-14T03:42:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 518 (ObjNum 371 snap 122 seq_num 371) dirty exists 2024-09-14T03:42:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:1447: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:42:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:1448: done (4 left) 2024-09-14T03:42:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:1449: done (3 left) 2024-09-14T03:42:29.720 INFO:tasks.rados.rados.0.smithi110.stdout:1453: snap_remove snap 111 2024-09-14T03:42:30.671 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:30.690 INFO:tasks.rados.rados.0.smithi110.stdout:1453: done (3 left) 2024-09-14T03:42:30.691 INFO:tasks.rados.rados.0.smithi110.stdout:1454: copy_from oid 35 from oid 46 current snap is 150 2024-09-14T03:42:30.691 INFO:tasks.rados.rados.0.smithi110.stdout:1455: copy_from oid 36 from oid 45 current snap is 150 2024-09-14T03:42:30.691 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 150 2024-09-14T03:42:30.691 INFO:tasks.rados.rados.0.smithi110.stdout:1456: seq_num 463 ranges {0=557056} 2024-09-14T03:42:30.694 INFO:tasks.rados.rados.0.smithi110.stdout:1456: writing smithi11075562-30 from 0 to 557056 tid 1 2024-09-14T03:42:30.694 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 150 2024-09-14T03:42:30.695 INFO:tasks.rados.rados.0.smithi110.stdout:1457: seq_num 464 ranges {6840320=573440} 2024-09-14T03:42:30.736 INFO:tasks.rados.rados.0.smithi110.stdout:1457: writing smithi11075562-27 from 6840320 to 7413760 tid 1 2024-09-14T03:42:30.736 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 150 2024-09-14T03:42:30.736 INFO:tasks.rados.rados.0.smithi110.stdout:1451: finishing write tid 1 to smithi11075562-12 2024-09-14T03:42:30.736 INFO:tasks.rados.rados.0.smithi110.stdout:1447: finishing copy_from to smithi11075562-34 2024-09-14T03:42:30.736 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 388 (ObjNum 446 snap 144 seq_num 446) dirty exists 2024-09-14T03:42:30.736 INFO:tasks.rados.rados.0.smithi110.stdout:1451: finishing write tid 2 to smithi11075562-12 2024-09-14T03:42:30.737 INFO:tasks.rados.rados.0.smithi110.stdout:1458: seq_num 465 ranges {999424=598016} 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1458: writing smithi11075562-11 from 999424 to 1597440 tid 1 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1447: done (7 left) 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1459: rmattr oid 24 current snap is 150 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1451: finishing write tid 3 to smithi11075562-12 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1000 (ObjNum 462 snap 150 seq_num 462) dirty exists 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1451: left oid 12 (ObjNum 462 snap 150 seq_num 462) 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1451: done (7 left) 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1460: delete oid 6 current snap is 150 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1452: finishing copy_from racing read to smithi11075562-28 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:1452: finishing copy_from to smithi11075562-28 2024-09-14T03:42:30.745 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 678 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1456: finishing write tid 1 to smithi11075562-30 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1456: finishing write tid 2 to smithi11075562-30 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1456: finishing write tid 3 to smithi11075562-30 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 896 (ObjNum 463 snap 150 seq_num 463) dirty exists 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1456: left oid 30 (ObjNum 463 snap 150 seq_num 463) 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1454: finishing copy_from racing read to smithi11075562-35 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1454: finishing copy_from to smithi11075562-35 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 573 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1455: finishing copy_from racing read to smithi11075562-36 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1455: finishing copy_from to smithi11075562-36 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 988 (ObjNum 457 snap 149 seq_num 457) dirty exists 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1457: finishing write tid 1 to smithi11075562-27 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1457: finishing write tid 2 to smithi11075562-27 2024-09-14T03:42:30.746 INFO:tasks.rados.rados.0.smithi110.stdout:1457: finishing write tid 3 to smithi11075562-27 2024-09-14T03:42:30.747 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 682 (ObjNum 464 snap 150 seq_num 464) dirty exists 2024-09-14T03:42:30.747 INFO:tasks.rados.rados.0.smithi110.stdout:1457: left oid 27 (ObjNum 464 snap 150 seq_num 464) 2024-09-14T03:42:30.747 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 519 (ObjNum 432 snap 138 seq_num 432) dirty exists 2024-09-14T03:42:30.747 INFO:tasks.rados.rados.0.smithi110.stdout:1452: done (7 left) 2024-09-14T03:42:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:1454: done (6 left) 2024-09-14T03:42:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:1455: done (5 left) 2024-09-14T03:42:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:1456: done (4 left) 2024-09-14T03:42:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:1457: done (3 left) 2024-09-14T03:42:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:1459: done (2 left) 2024-09-14T03:42:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:1460: done (1 left) 2024-09-14T03:42:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 150 2024-09-14T03:42:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:1461: seq_num 467 ranges {491520=630784} 2024-09-14T03:42:30.753 INFO:tasks.rados.rados.0.smithi110.stdout:1461: writing smithi11075562-5 from 491520 to 1122304 tid 1 2024-09-14T03:42:30.753 INFO:tasks.rados.rados.0.smithi110.stdout:1462: delete oid 38 current snap is 150 2024-09-14T03:42:30.753 INFO:tasks.rados.rados.0.smithi110.stdout:1458: finishing write tid 1 to smithi11075562-11 2024-09-14T03:42:30.753 INFO:tasks.rados.rados.0.smithi110.stdout:1458: finishing write tid 2 to smithi11075562-11 2024-09-14T03:42:30.753 INFO:tasks.rados.rados.0.smithi110.stdout:1458: finishing write tid 3 to smithi11075562-11 2024-09-14T03:42:30.753 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 886 (ObjNum 465 snap 150 seq_num 465) dirty exists 2024-09-14T03:42:30.753 INFO:tasks.rados.rados.0.smithi110.stdout:1458: left oid 11 (ObjNum 465 snap 150 seq_num 465) 2024-09-14T03:42:30.755 INFO:tasks.rados.rados.0.smithi110.stdout:1458: done (2 left) 2024-09-14T03:42:30.755 INFO:tasks.rados.rados.0.smithi110.stdout:1462: done (1 left) 2024-09-14T03:42:30.755 INFO:tasks.rados.rados.0.smithi110.stdout:1463: setattr oid 17 current snap is 150 2024-09-14T03:42:30.756 INFO:tasks.rados.rados.0.smithi110.stdout:1464: read oid 21 snap -1 2024-09-14T03:42:30.756 INFO:tasks.rados.rados.0.smithi110.stdout:1464: expect (ObjNum 442 snap 142 seq_num 442) 2024-09-14T03:42:30.756 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 150 2024-09-14T03:42:30.756 INFO:tasks.rados.rados.0.smithi110.stdout:1465: seq_num 469 ranges {876544=663552} 2024-09-14T03:42:30.765 INFO:tasks.rados.rados.0.smithi110.stdout:1465: writing smithi11075562-26 from 876544 to 1540096 tid 1 2024-09-14T03:42:30.765 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 150 2024-09-14T03:42:30.765 INFO:tasks.rados.rados.0.smithi110.stdout:1461: finishing write tid 1 to smithi11075562-5 2024-09-14T03:42:30.765 INFO:tasks.rados.rados.0.smithi110.stdout:1466: seq_num 470 ranges {442368=679936} 2024-09-14T03:42:30.773 INFO:tasks.rados.rados.0.smithi110.stdout:1466: writing smithi11075562-7 from 442368 to 1122304 tid 1 2024-09-14T03:42:30.773 INFO:tasks.rados.rados.0.smithi110.stdout:1467: rollback oid 34 current snap is 150 2024-09-14T03:42:30.774 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 142 2024-09-14T03:42:30.774 INFO:tasks.rados.rados.0.smithi110.stdout:1461: finishing write tid 2 to smithi11075562-5 2024-09-14T03:42:30.774 INFO:tasks.rados.rados.0.smithi110.stdout:1461: finishing write tid 3 to smithi11075562-5 2024-09-14T03:42:30.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1009 (ObjNum 467 snap 150 seq_num 467) dirty exists 2024-09-14T03:42:30.774 INFO:tasks.rados.rados.0.smithi110.stdout:1461: left oid 5 (ObjNum 467 snap 150 seq_num 467) 2024-09-14T03:42:30.775 INFO:tasks.rados.rados.0.smithi110.stdout:1461: done (5 left) 2024-09-14T03:42:30.775 INFO:tasks.rados.rados.0.smithi110.stdout:1468: rollback oid 13 current snap is 150 2024-09-14T03:42:30.775 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 129 2024-09-14T03:42:30.775 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 887 (ObjNum 33 snap 0 seq_num 64961024) dirty exists 2024-09-14T03:42:30.775 INFO:tasks.rados.rados.0.smithi110.stdout:1463: done (5 left) 2024-09-14T03:42:30.775 INFO:tasks.rados.rados.0.smithi110.stdout:1469: snap_create 2024-09-14T03:42:30.776 INFO:tasks.rados.rados.0.smithi110.stdout:1464: expect (ObjNum 442 snap 142 seq_num 442) 2024-09-14T03:42:30.853 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.853 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[88262]: osdmap e354: 8 total, 7 up, 8 in 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[88262]: pgmap v157: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 420 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.4 MiB/s wr, 7 op/s; 99/849 objects degraded (11.661%) 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[86423]: osdmap e354: 8 total, 7 up, 8 in 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[86423]: pgmap v157: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 420 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.4 MiB/s wr, 7 op/s; 99/849 objects degraded (11.661%) 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.854 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:30 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.899 INFO:tasks.rados.rados.0.smithi110.stdout:1465: finishing write tid 1 to smithi11075562-26 2024-09-14T03:42:30.899 INFO:tasks.rados.rados.0.smithi110.stdout:1465: finishing write tid 2 to smithi11075562-26 2024-09-14T03:42:30.899 INFO:tasks.rados.rados.0.smithi110.stdout:1465: finishing write tid 3 to smithi11075562-26 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 759 (ObjNum 469 snap 150 seq_num 469) dirty exists 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1465: left oid 26 (ObjNum 469 snap 150 seq_num 469) 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1467: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1467: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1466: finishing write tid 1 to smithi11075562-7 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1466: finishing write tid 2 to smithi11075562-7 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1467: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 507 (ObjNum 387 snap 127 seq_num 387) dirty exists 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1466: finishing write tid 3 to smithi11075562-7 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 487 (ObjNum 470 snap 150 seq_num 470) dirty exists 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1466: left oid 7 (ObjNum 470 snap 150 seq_num 470) 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1468: finishing rollback tid 0 to smithi11075562-13 2024-09-14T03:42:30.900 INFO:tasks.rados.rados.0.smithi110.stdout:1468: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:42:30.901 INFO:tasks.rados.rados.0.smithi110.stdout:1468: finishing rollback tid 2 to smithi11075562-13 2024-09-14T03:42:30.901 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 891 (ObjNum 356 snap 115 seq_num 356) dirty exists 2024-09-14T03:42:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:30 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:30 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:30 smithi195 ceph-mon[82388]: osdmap e354: 8 total, 7 up, 8 in 2024-09-14T03:42:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:30 smithi195 ceph-mon[82388]: pgmap v157: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 420 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.4 MiB/s wr, 7 op/s; 99/849 objects degraded (11.661%) 2024-09-14T03:42:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:30 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:30 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:31.691 INFO:tasks.rados.rados.0.smithi110.stdout:1464: done (5 left) 2024-09-14T03:42:31.692 INFO:tasks.rados.rados.0.smithi110.stdout:1465: done (4 left) 2024-09-14T03:42:31.692 INFO:tasks.rados.rados.0.smithi110.stdout:1466: done (3 left) 2024-09-14T03:42:31.692 INFO:tasks.rados.rados.0.smithi110.stdout:1467: done (2 left) 2024-09-14T03:42:31.692 INFO:tasks.rados.rados.0.smithi110.stdout:1468: done (1 left) 2024-09-14T03:42:31.692 INFO:tasks.rados.rados.0.smithi110.stdout:1469: done (0 left) 2024-09-14T03:42:31.692 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 151 2024-09-14T03:42:31.692 INFO:tasks.rados.rados.0.smithi110.stdout:1470: seq_num 471 ranges {1122304=696320} 2024-09-14T03:42:31.701 INFO:tasks.rados.rados.0.smithi110.stdout:1470: writing smithi11075562-5 from 1122304 to 1818624 tid 1 2024-09-14T03:42:31.701 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 151 2024-09-14T03:42:31.701 INFO:tasks.rados.rados.0.smithi110.stdout:1471: seq_num 472 ranges {6807552=712704} 2024-09-14T03:42:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:1471: writing smithi11075562-10 from 6807552 to 7520256 tid 1 2024-09-14T03:42:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:1472: rmattr oid 30 current snap is 151 2024-09-14T03:42:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:1472: done (2 left) 2024-09-14T03:42:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:1473: copy_from oid 33 from oid 50 current snap is 151 2024-09-14T03:42:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:1470: finishing write tid 1 to smithi11075562-5 2024-09-14T03:42:31.743 INFO:tasks.rados.rados.0.smithi110.stdout:1470: finishing write tid 2 to smithi11075562-5 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:1470: finishing write tid 3 to smithi11075562-5 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1012 (ObjNum 471 snap 151 seq_num 471) dirty exists 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:1470: left oid 5 (ObjNum 471 snap 151 seq_num 471) 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:1470: done (2 left) 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:1474: read oid 28 snap -1 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:1474: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:1475: rollback oid 11 current snap is 151 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 147 2024-09-14T03:42:31.744 INFO:tasks.rados.rados.0.smithi110.stdout:1476: snap_create 2024-09-14T03:42:31.746 INFO:tasks.rados.rados.0.smithi110.stdout:1475: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:42:31.747 INFO:tasks.rados.rados.0.smithi110.stdout:1475: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:42:31.747 INFO:tasks.rados.rados.0.smithi110.stdout:1475: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:42:31.747 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 890 (ObjNum 454 snap 146 seq_num 454) dirty exists 2024-09-14T03:42:31.750 INFO:tasks.rados.rados.0.smithi110.stdout:1471: finishing write tid 1 to smithi11075562-10 2024-09-14T03:42:31.750 INFO:tasks.rados.rados.0.smithi110.stdout:1471: finishing write tid 2 to smithi11075562-10 2024-09-14T03:42:31.752 INFO:tasks.rados.rados.0.smithi110.stdout:1471: finishing write tid 3 to smithi11075562-10 2024-09-14T03:42:31.752 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 521 (ObjNum 472 snap 151 seq_num 472) dirty exists 2024-09-14T03:42:31.752 INFO:tasks.rados.rados.0.smithi110.stdout:1471: left oid 10 (ObjNum 472 snap 151 seq_num 472) 2024-09-14T03:42:31.771 INFO:tasks.rados.rados.0.smithi110.stdout:1474: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:42:31.875 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:42:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:42:31] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.51.0" 2024-09-14T03:42:31.876 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[97937]: 2024-09-14T03:42:31.783+0000 7f24ad37c640 -1 osd.1 344 log_to_monitors true 2024-09-14T03:42:31.876 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:31 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 99/849 objects degraded (11.661%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:31.876 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:31 smithi110 ceph-mon[88262]: osdmap e355: 8 total, 7 up, 8 in 2024-09-14T03:42:31.876 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:31 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 99/849 objects degraded (11.661%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:31.877 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:31 smithi110 ceph-mon[86423]: osdmap e355: 8 total, 7 up, 8 in 2024-09-14T03:42:31.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:31 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 99/849 objects degraded (11.661%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:31.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:31 smithi195 ceph-mon[82388]: osdmap e355: 8 total, 7 up, 8 in 2024-09-14T03:42:32.091 INFO:tasks.rados.rados.0.smithi110.stdout:1473: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:42:32.091 INFO:tasks.rados.rados.0.smithi110.stdout:1473: finishing copy_from to smithi11075562-33 2024-09-14T03:42:32.091 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 929 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:32.699 INFO:tasks.rados.rados.0.smithi110.stdout:1471: done (4 left) 2024-09-14T03:42:32.699 INFO:tasks.rados.rados.0.smithi110.stdout:1473: done (3 left) 2024-09-14T03:42:32.699 INFO:tasks.rados.rados.0.smithi110.stdout:1474: done (2 left) 2024-09-14T03:42:32.699 INFO:tasks.rados.rados.0.smithi110.stdout:1475: done (1 left) 2024-09-14T03:42:32.699 INFO:tasks.rados.rados.0.smithi110.stdout:1476: done (0 left) 2024-09-14T03:42:32.699 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 152 2024-09-14T03:42:32.699 INFO:tasks.rados.rados.0.smithi110.stdout:1477: seq_num 473 ranges {999424=729088} 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1477: writing smithi11075562-11 from 999424 to 1728512 tid 1 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1478: rollback oid 15 current snap is 152 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 143 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1479: read oid 23 snap -1 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1479: expect (ObjNum 371 snap 122 seq_num 371) 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1480: rollback oid 40 current snap is 152 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 40 to 67 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1481: read oid 35 snap -1 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1481: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:42:32.710 INFO:tasks.rados.rados.0.smithi110.stdout:1482: read oid 1 snap -1 2024-09-14T03:42:32.711 INFO:tasks.rados.rados.0.smithi110.stdout:1482: expect (ObjNum 424 snap 136 seq_num 424) 2024-09-14T03:42:32.711 INFO:tasks.rados.rados.0.smithi110.stdout:1483: delete oid 26 current snap is 152 2024-09-14T03:42:32.720 INFO:tasks.rados.rados.0.smithi110.stdout:1478: finishing rollback tid 0 to smithi11075562-15 2024-09-14T03:42:32.720 INFO:tasks.rados.rados.0.smithi110.stdout:1478: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:42:32.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 901 (ObjNum 33 snap 0 seq_num 0) dirty dne 2024-09-14T03:42:32.720 INFO:tasks.rados.rados.0.smithi110.stdout:1477: finishing write tid 1 to smithi11075562-11 2024-09-14T03:42:32.720 INFO:tasks.rados.rados.0.smithi110.stdout:1477: finishing write tid 2 to smithi11075562-11 2024-09-14T03:42:32.721 INFO:tasks.rados.rados.0.smithi110.stdout:1478: done (6 left) 2024-09-14T03:42:32.721 INFO:tasks.rados.rados.0.smithi110.stdout:1483: done (5 left) 2024-09-14T03:42:32.721 INFO:tasks.rados.rados.0.smithi110.stdout:1484: delete oid 19 current snap is 152 2024-09-14T03:42:32.722 INFO:tasks.rados.rados.0.smithi110.stdout:1480: finishing rollback tid 0 to smithi11075562-40 2024-09-14T03:42:32.722 INFO:tasks.rados.rados.0.smithi110.stdout:1480: finishing rollback tid 1 to smithi11075562-40 2024-09-14T03:42:32.722 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 783 (ObjNum 544235886 snap 2608 seq_num 304) dirty dne 2024-09-14T03:42:32.724 INFO:tasks.rados.rados.0.smithi110.stdout:1481: expect (ObjNum 321 snap 104 seq_num 321) 2024-09-14T03:42:32.747 INFO:tasks.rados.rados.0.smithi110.stdout:1477: finishing write tid 3 to smithi11075562-11 2024-09-14T03:42:32.747 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 892 (ObjNum 473 snap 152 seq_num 473) dirty exists 2024-09-14T03:42:32.747 INFO:tasks.rados.rados.0.smithi110.stdout:1477: left oid 11 (ObjNum 473 snap 152 seq_num 473) 2024-09-14T03:42:32.747 INFO:tasks.rados.rados.0.smithi110.stdout:1477: done (5 left) 2024-09-14T03:42:32.748 INFO:tasks.rados.rados.0.smithi110.stdout:1480: done (4 left) 2024-09-14T03:42:32.748 INFO:tasks.rados.rados.0.smithi110.stdout:1481: done (3 left) 2024-09-14T03:42:32.748 INFO:tasks.rados.rados.0.smithi110.stdout:1484: done (2 left) 2024-09-14T03:42:32.748 INFO:tasks.rados.rados.0.smithi110.stdout:1485: copy_from oid 15 from oid 31 current snap is 152 2024-09-14T03:42:32.748 INFO:tasks.rados.rados.0.smithi110.stdout:1486: snap_remove snap 149 2024-09-14T03:42:32.930 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-mon[88262]: osdmap e356: 8 total, 7 up, 8 in 2024-09-14T03:42:32.930 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-mon[88262]: pgmap v160: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 420 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 220 KiB/s wr, 1 op/s; 99/849 objects degraded (11.661%) 2024-09-14T03:42:32.930 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-mon[88262]: from='osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:42:32.931 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-mon[88262]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:42:32.931 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[97937]: 2024-09-14T03:42:32.723+0000 7f24a5126640 -1 osd.1 344 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:42:32.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-mon[86423]: osdmap e356: 8 total, 7 up, 8 in 2024-09-14T03:42:32.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-mon[86423]: pgmap v160: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 420 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 220 KiB/s wr, 1 op/s; 99/849 objects degraded (11.661%) 2024-09-14T03:42:32.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-mon[86423]: from='osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:42:32.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:32 smithi110 ceph-mon[86423]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:42:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:32 smithi195 ceph-mon[82388]: osdmap e356: 8 total, 7 up, 8 in 2024-09-14T03:42:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:32 smithi195 ceph-mon[82388]: pgmap v160: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 420 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 220 KiB/s wr, 1 op/s; 99/849 objects degraded (11.661%) 2024-09-14T03:42:32.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:32 smithi195 ceph-mon[82388]: from='osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:42:32.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:32 smithi195 ceph-mon[82388]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:42:33.697 INFO:tasks.rados.rados.0.smithi110.stdout:1479: expect (ObjNum 371 snap 122 seq_num 371) 2024-09-14T03:42:33.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:42:33.772+0000 7f940e88b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:42:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:42:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: osdmap e357: 8 total, 7 up, 8 in 2024-09-14T03:42:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: from='osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: osdmap e357: 8 total, 7 up, 8 in 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: from='osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:33.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:33.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:42:33.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:33 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: osdmap e357: 8 total, 7 up, 8 in 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: from='osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:42:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:33 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:34.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:34.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:42:34.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:34.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:34.686 INFO:tasks.rados.rados.0.smithi110.stdout:1485: finishing copy_from to smithi11075562-15 2024-09-14T03:42:34.686 INFO:tasks.rados.rados.0.smithi110.stdout:1485: got expected ENOENT (src dne) 2024-09-14T03:42:34.686 INFO:tasks.rados.rados.0.smithi110.stdout:1485: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:42:34.686 INFO:tasks.rados.rados.0.smithi110.stdout:1482: expect (ObjNum 424 snap 136 seq_num 424) 2024-09-14T03:42:35.163 INFO:tasks.rados.rados.0.smithi110.stdout:1479: done (3 left) 2024-09-14T03:42:35.164 INFO:tasks.rados.rados.0.smithi110.stdout:1482: done (2 left) 2024-09-14T03:42:35.164 INFO:tasks.rados.rados.0.smithi110.stdout:1485: done (1 left) 2024-09-14T03:42:35.164 INFO:tasks.rados.rados.0.smithi110.stdout:1486: done (0 left) 2024-09-14T03:42:35.164 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 152 2024-09-14T03:42:35.164 INFO:tasks.rados.rados.0.smithi110.stdout:1487: seq_num 476 ranges {6365184=778240} 2024-09-14T03:42:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:42:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418] boot 2024-09-14T03:42:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:42:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: pgmap v163: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 454 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 7.3 MiB/s wr, 23 op/s; 105/915 objects degraded (11.475%) 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418] boot 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:42:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:42:35.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: pgmap v163: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 454 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 7.3 MiB/s wr, 23 op/s; 105/915 objects degraded (11.475%) 2024-09-14T03:42:35.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:35.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:34 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:42:35.201 INFO:tasks.rados.rados.0.smithi110.stdout:1487: writing smithi11075562-33 from 6365184 to 7143424 tid 1 2024-09-14T03:42:35.201 INFO:tasks.rados.rados.0.smithi110.stdout:1488: copy_from oid 22 from oid 50 current snap is 152 2024-09-14T03:42:35.201 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 152 2024-09-14T03:42:35.201 INFO:tasks.rados.rados.0.smithi110.stdout:1489: seq_num 477 ranges {0=794624} 2024-09-14T03:42:35.207 INFO:tasks.rados.rados.0.smithi110.stdout:1489: writing smithi11075562-38 from 0 to 794624 tid 1 2024-09-14T03:42:35.207 INFO:tasks.rados.rados.0.smithi110.stdout:1490: read oid 17 snap -1 2024-09-14T03:42:35.207 INFO:tasks.rados.rados.0.smithi110.stdout:1490: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:42:35.207 INFO:tasks.rados.rados.0.smithi110.stdout:1487: finishing write tid 1 to smithi11075562-33 2024-09-14T03:42:35.207 INFO:tasks.rados.rados.0.smithi110.stdout:1487: finishing write tid 2 to smithi11075562-33 2024-09-14T03:42:35.207 INFO:tasks.rados.rados.0.smithi110.stdout:1487: finishing write tid 3 to smithi11075562-33 2024-09-14T03:42:35.207 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 931 (ObjNum 476 snap 152 seq_num 476) dirty exists 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1487: left oid 33 (ObjNum 476 snap 152 seq_num 476) 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1491: read oid 27 snap 83 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1491: expect (ObjNum 64958384 snap 21934 seq_num 66776688) 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1487: done (4 left) 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1492: rmattr oid 36 current snap is 152 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1492: done (4 left) 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1493: rollback oid 13 current snap is 152 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 145 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1494: read oid 33 snap -1 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1494: expect (ObjNum 476 snap 152 seq_num 476) 2024-09-14T03:42:35.208 INFO:tasks.rados.rados.0.smithi110.stdout:1495: snap_remove snap 143 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: osd.1 [v2:172.21.15.110:6810/1059959418,v1:172.21.15.110:6811/1059959418] boot 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: pgmap v163: 121 pgs: 33 active+undersized, 11 active+undersized+degraded, 77 active+clean; 454 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 7.3 MiB/s wr, 23 op/s; 105/915 objects degraded (11.475%) 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:34 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:42:35.721 INFO:tasks.rados.rados.0.smithi110.stdout:1495: done (6 left) 2024-09-14T03:42:35.721 INFO:tasks.rados.rados.0.smithi110.stdout:1496: setattr oid 35 current snap is 152 2024-09-14T03:42:35.721 INFO:tasks.rados.rados.0.smithi110.stdout:1490: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:42:35.724 INFO:tasks.rados.rados.0.smithi110.stdout:1491: expect (ObjNum 64958384 snap 21934 seq_num 66776688) 2024-09-14T03:42:35.726 INFO:tasks.rados.rados.0.smithi110.stdout:1493: finishing rollback tid 0 to smithi11075562-13 2024-09-14T03:42:35.726 INFO:tasks.rados.rados.0.smithi110.stdout:1493: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:42:35.726 INFO:tasks.rados.rados.0.smithi110.stdout:1489: finishing write tid 1 to smithi11075562-38 2024-09-14T03:42:35.726 INFO:tasks.rados.rados.0.smithi110.stdout:1489: finishing write tid 2 to smithi11075562-38 2024-09-14T03:42:35.726 INFO:tasks.rados.rados.0.smithi110.stdout:1493: finishing rollback tid 2 to smithi11075562-13 2024-09-14T03:42:35.727 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 899 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:42:35.727 INFO:tasks.rados.rados.0.smithi110.stdout:1489: finishing write tid 3 to smithi11075562-38 2024-09-14T03:42:35.727 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 489 (ObjNum 477 snap 152 seq_num 477) dirty exists 2024-09-14T03:42:35.727 INFO:tasks.rados.rados.0.smithi110.stdout:1489: left oid 38 (ObjNum 477 snap 152 seq_num 477) 2024-09-14T03:42:35.727 INFO:tasks.rados.rados.0.smithi110.stdout:1488: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:42:35.727 INFO:tasks.rados.rados.0.smithi110.stdout:1488: finishing copy_from to smithi11075562-22 2024-09-14T03:42:35.727 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 929 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:35.727 INFO:tasks.rados.rados.0.smithi110.stdout:1494: expect (ObjNum 476 snap 152 seq_num 476) 2024-09-14T03:42:36.160 INFO:tasks.rados.rados.0.smithi110.stdout:1488: done (6 left) 2024-09-14T03:42:36.160 INFO:tasks.rados.rados.0.smithi110.stdout:1489: done (5 left) 2024-09-14T03:42:36.160 INFO:tasks.rados.rados.0.smithi110.stdout:1490: done (4 left) 2024-09-14T03:42:36.160 INFO:tasks.rados.rados.0.smithi110.stdout:1491: done (3 left) 2024-09-14T03:42:36.160 INFO:tasks.rados.rados.0.smithi110.stdout:1493: done (2 left) 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1494: done (1 left) 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1497: copy_from oid 12 from oid 18 current snap is 152 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1498: read oid 24 snap -1 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1498: expect (ObjNum 432 snap 138 seq_num 432) 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1499: read oid 23 snap -1 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1499: expect (ObjNum 371 snap 122 seq_num 371) 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1500: copy_from oid 17 from oid 42 current snap is 152 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1501: read oid 20 snap 133 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1501: expect (ObjNum 292 snap 90 seq_num 292) 2024-09-14T03:42:36.161 INFO:tasks.rados.rados.0.smithi110.stdout:1502: snap_create 2024-09-14T03:42:36.163 INFO:tasks.rados.rados.0.smithi110.stdout:1501: expect (ObjNum 292 snap 90 seq_num 292) 2024-09-14T03:42:36.169 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 907 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:42:36.181 INFO:tasks.rados.rados.0.smithi110.stdout:1497: finishing copy_from racing read to smithi11075562-12 2024-09-14T03:42:36.184 INFO:tasks.rados.rados.0.smithi110.stdout:1497: finishing copy_from to smithi11075562-12 2024-09-14T03:42:36.184 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 640 (ObjNum 407 snap 134 seq_num 407) dirty exists 2024-09-14T03:42:36.187 INFO:tasks.rados.rados.0.smithi110.stdout:1498: expect (ObjNum 432 snap 138 seq_num 432) 2024-09-14T03:42:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:35 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:42:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:35 smithi110 ceph-mon[88262]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:42:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:35 smithi110 ceph-mon[88262]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:42:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:35 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:42:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:35 smithi110 ceph-mon[86423]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:42:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:35 smithi110 ceph-mon[86423]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:42:36.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:35 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:42:36.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:35 smithi195 ceph-mon[82388]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:42:36.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:35 smithi195 ceph-mon[82388]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:42:36.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:35.739Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:36.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:36.605 INFO:tasks.rados.rados.0.smithi110.stdout:1500: finishing copy_from racing read to smithi11075562-17 2024-09-14T03:42:36.605 INFO:tasks.rados.rados.0.smithi110.stdout:1500: finishing copy_from to smithi11075562-17 2024-09-14T03:42:36.605 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 678 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:36.605 INFO:tasks.rados.rados.0.smithi110.stdout:1499: expect (ObjNum 371 snap 122 seq_num 371) 2024-09-14T03:42:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:36 smithi110 ceph-mon[86423]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:42:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:36 smithi110 ceph-mon[86423]: pgmap v166: 121 pgs: 6 peering, 28 active+undersized, 10 active+undersized+degraded, 77 active+clean; 461 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.9 MiB/s wr, 25 op/s; 86/918 objects degraded (9.368%) 2024-09-14T03:42:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:36 smithi110 ceph-mon[88262]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:42:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:36 smithi110 ceph-mon[88262]: pgmap v166: 121 pgs: 6 peering, 28 active+undersized, 10 active+undersized+degraded, 77 active+clean; 461 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.9 MiB/s wr, 25 op/s; 86/918 objects degraded (9.368%) 2024-09-14T03:42:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:36 smithi195 ceph-mon[82388]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:42:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:36 smithi195 ceph-mon[82388]: pgmap v166: 121 pgs: 6 peering, 28 active+undersized, 10 active+undersized+degraded, 77 active+clean; 461 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.9 MiB/s wr, 25 op/s; 86/918 objects degraded (9.368%) 2024-09-14T03:42:37.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:37.574 INFO:tasks.rados.rados.0.smithi110.stdout:1496: done (6 left) 2024-09-14T03:42:37.574 INFO:tasks.rados.rados.0.smithi110.stdout:1497: done (5 left) 2024-09-14T03:42:37.574 INFO:tasks.rados.rados.0.smithi110.stdout:1498: done (4 left) 2024-09-14T03:42:37.574 INFO:tasks.rados.rados.0.smithi110.stdout:1499: done (3 left) 2024-09-14T03:42:37.574 INFO:tasks.rados.rados.0.smithi110.stdout:1500: done (2 left) 2024-09-14T03:42:37.574 INFO:tasks.rados.rados.0.smithi110.stdout:1501: done (1 left) 2024-09-14T03:42:37.574 INFO:tasks.rados.rados.0.smithi110.stdout:1502: done (0 left) 2024-09-14T03:42:37.574 INFO:tasks.rados.rados.0.smithi110.stdout:1503: snap_create 2024-09-14T03:42:37.729 INFO:tasks.rados.rados.0.smithi110.stdout:1503: done (0 left) 2024-09-14T03:42:37.729 INFO:tasks.rados.rados.0.smithi110.stdout:1504: setattr oid 9 current snap is 154 2024-09-14T03:42:37.730 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 154 2024-09-14T03:42:37.730 INFO:tasks.rados.rados.0.smithi110.stdout:1505: seq_num 478 ranges {0=409600,409600=401408} 2024-09-14T03:42:37.732 INFO:tasks.rados.rados.0.smithi110.stdout:1505: writing smithi11075562-40 from 0 to 409600 tid 1 2024-09-14T03:42:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1505: writing smithi11075562-40 from 409600 to 811008 tid 2 2024-09-14T03:42:37.735 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 154 2024-09-14T03:42:37.735 INFO:tasks.rados.rados.0.smithi110.stdout:1506: seq_num 479 ranges {12058624=425984,12484608=401408} 2024-09-14T03:42:37.801 INFO:tasks.rados.rados.0.smithi110.stdout:1506: writing smithi11075562-23 from 12058624 to 12484608 tid 1 2024-09-14T03:42:37.803 INFO:tasks.rados.rados.0.smithi110.stdout:1506: writing smithi11075562-23 from 12484608 to 12886016 tid 2 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 154 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 932 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:1505: finishing write tid 1 to smithi11075562-40 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:1505: finishing write tid 2 to smithi11075562-40 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:1505: finishing write tid 3 to smithi11075562-40 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:1505: finishing write tid 4 to smithi11075562-40 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 795 (ObjNum 478 snap 154 seq_num 478) dirty exists 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:1505: left oid 40 (ObjNum 478 snap 154 seq_num 478) 2024-09-14T03:42:37.804 INFO:tasks.rados.rados.0.smithi110.stdout:1507: seq_num 480 ranges {794624=442368,1236992=401408} 2024-09-14T03:42:37.810 INFO:tasks.rados.rados.0.smithi110.stdout:1507: writing smithi11075562-38 from 794624 to 1236992 tid 1 2024-09-14T03:42:37.813 INFO:tasks.rados.rados.0.smithi110.stdout:1507: writing smithi11075562-38 from 1236992 to 1638400 tid 2 2024-09-14T03:42:37.813 INFO:tasks.rados.rados.0.smithi110.stdout:1504: done (3 left) 2024-09-14T03:42:37.813 INFO:tasks.rados.rados.0.smithi110.stdout:1505: done (2 left) 2024-09-14T03:42:37.813 INFO:tasks.rados.rados.0.smithi110.stdout:1508: read oid 48 snap 139 2024-09-14T03:42:37.813 INFO:tasks.rados.rados.0.smithi110.stdout:1508: expect (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:42:37.813 INFO:tasks.rados.rados.0.smithi110.stdout:1509: rmattr oid 43 current snap is 154 2024-09-14T03:42:37.813 INFO:tasks.rados.rados.0.smithi110.stdout:1509: done (3 left) 2024-09-14T03:42:37.813 INFO:tasks.rados.rados.0.smithi110.stdout:1510: snap_remove snap 141 2024-09-14T03:42:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:37 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 86/918 objects degraded (9.368%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:37 smithi110 ceph-mon[86423]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:42:38.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:37 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 86/918 objects degraded (9.368%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:38.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:37 smithi110 ceph-mon[88262]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:42:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:37 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 86/918 objects degraded (9.368%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:37 smithi195 ceph-mon[82388]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:42:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:1510: done (3 left) 2024-09-14T03:42:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 154 2024-09-14T03:42:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:1506: finishing write tid 1 to smithi11075562-23 2024-09-14T03:42:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:1506: finishing write tid 2 to smithi11075562-23 2024-09-14T03:42:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:1506: finishing write tid 3 to smithi11075562-23 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:1506: finishing write tid 4 to smithi11075562-23 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 524 (ObjNum 479 snap 154 seq_num 479) dirty exists 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:1506: left oid 23 (ObjNum 479 snap 154 seq_num 479) 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:1507: finishing write tid 1 to smithi11075562-38 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:1507: finishing write tid 2 to smithi11075562-38 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:1507: finishing write tid 3 to smithi11075562-38 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:1507: finishing write tid 4 to smithi11075562-38 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 492 (ObjNum 480 snap 154 seq_num 480) dirty exists 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:1507: left oid 38 (ObjNum 480 snap 154 seq_num 480) 2024-09-14T03:42:38.738 INFO:tasks.rados.rados.0.smithi110.stdout:1508: expect (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:42:39.013 INFO:tasks.rados.rados.0.smithi110.stdout:1511: seq_num 481 ranges {581632=458752,1040384=401408} 2024-09-14T03:42:39.018 INFO:tasks.rados.rados.0.smithi110.stdout:1511: writing smithi11075562-46 from 581632 to 1040384 tid 1 2024-09-14T03:42:39.021 INFO:tasks.rados.rados.0.smithi110.stdout:1511: writing smithi11075562-46 from 1040384 to 1441792 tid 2 2024-09-14T03:42:39.021 INFO:tasks.rados.rados.0.smithi110.stdout:1506: done (3 left) 2024-09-14T03:42:39.021 INFO:tasks.rados.rados.0.smithi110.stdout:1507: done (2 left) 2024-09-14T03:42:39.021 INFO:tasks.rados.rados.0.smithi110.stdout:1508: done (1 left) 2024-09-14T03:42:39.021 INFO:tasks.rados.rados.0.smithi110.stdout:1512: read oid 43 snap -1 2024-09-14T03:42:39.021 INFO:tasks.rados.rados.0.smithi110.stdout:1512: expect (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:42:39.021 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 154 2024-09-14T03:42:39.021 INFO:tasks.rados.rados.0.smithi110.stdout:1513: seq_num 482 ranges {0=475136,475136=401408} 2024-09-14T03:42:39.023 INFO:tasks.rados.rados.0.smithi110.stdout:1513: writing smithi11075562-9 from 0 to 475136 tid 1 2024-09-14T03:42:39.027 INFO:tasks.rados.rados.0.smithi110.stdout:1513: writing smithi11075562-9 from 475136 to 876544 tid 2 2024-09-14T03:42:39.027 INFO:tasks.rados.rados.0.smithi110.stdout:1514: read oid 41 snap -1 2024-09-14T03:42:39.027 INFO:tasks.rados.rados.0.smithi110.stdout:1514: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:42:39.027 INFO:tasks.rados.rados.0.smithi110.stdout:1515: rollback oid 26 current snap is 154 2024-09-14T03:42:39.027 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 126 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1511: finishing write tid 1 to smithi11075562-46 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1511: finishing write tid 2 to smithi11075562-46 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1511: finishing write tid 3 to smithi11075562-46 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1516: read oid 2 snap -1 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1516: expect (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1517: rollback oid 11 current snap is 154 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 152 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1518: rollback oid 28 current snap is 154 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 28 to 136 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1519: delete oid 19 current snap is 154 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1511: finishing write tid 4 to smithi11075562-46 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 935 (ObjNum 481 snap 154 seq_num 481) dirty exists 2024-09-14T03:42:39.028 INFO:tasks.rados.rados.0.smithi110.stdout:1511: left oid 46 (ObjNum 481 snap 154 seq_num 481) 2024-09-14T03:42:39.030 INFO:tasks.rados.rados.0.smithi110.stdout:1513: finishing write tid 1 to smithi11075562-9 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1511: done (8 left) 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1519: done (7 left) 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1520: read oid 32 snap -1 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1520: expect (ObjNum 449 snap 144 seq_num 449) 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1521: read oid 6 snap 67 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1521: expect (ObjNum 176 snap 50 seq_num 176) 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1522: read oid 10 snap 129 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1522: expect (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1523: copy_from oid 33 from oid 37 current snap is 154 2024-09-14T03:42:39.032 INFO:tasks.rados.rados.0.smithi110.stdout:1515: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1515: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 762 (ObjNum 365 snap 118 seq_num 365) dirty exists 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1513: finishing write tid 2 to smithi11075562-9 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1513: finishing write tid 3 to smithi11075562-9 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1517: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1515: done (10 left) 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1524: copy_from oid 46 from oid 42 current snap is 154 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1525: delete oid 19 current snap is 154 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1517: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:1517: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:42:39.033 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 895 (ObjNum 473 snap 152 seq_num 473) dirty exists 2024-09-14T03:42:39.034 INFO:tasks.rados.rados.0.smithi110.stdout:1513: finishing write tid 4 to smithi11075562-9 2024-09-14T03:42:39.034 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 938 (ObjNum 482 snap 154 seq_num 482) dirty exists 2024-09-14T03:42:39.034 INFO:tasks.rados.rados.0.smithi110.stdout:1513: left oid 9 (ObjNum 482 snap 154 seq_num 482) 2024-09-14T03:42:39.036 INFO:tasks.rados.rados.0.smithi110.stdout:1514: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:42:39.043 INFO:tasks.rados.rados.0.smithi110.stdout:1512: expect (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:42:39.194 INFO:tasks.rados.rados.0.smithi110.stdout:1518: finishing rollback tid 0 to smithi11075562-28 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1518: finishing rollback tid 1 to smithi11075562-28 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1518: finishing rollback tid 2 to smithi11075562-28 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 686 (ObjNum 386 snap 127 seq_num 386) dirty exists 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1512: done (11 left) 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1513: done (10 left) 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1514: done (9 left) 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1517: done (8 left) 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1518: done (7 left) 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1525: done (6 left) 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 154 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1524: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:42:39.195 INFO:tasks.rados.rados.0.smithi110.stdout:1521: expect (ObjNum 176 snap 50 seq_num 176) 2024-09-14T03:42:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:38 smithi110 ceph-mon[88262]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:42:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:38 smithi110 ceph-mon[88262]: pgmap v169: 121 pgs: 6 peering, 1 active+undersized, 114 active+clean; 473 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.2 MiB/s wr, 10 op/s; 5.9 MiB/s, 3 objects/s recovering 2024-09-14T03:42:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:38 smithi110 ceph-mon[86423]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:42:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:38 smithi110 ceph-mon[86423]: pgmap v169: 121 pgs: 6 peering, 1 active+undersized, 114 active+clean; 473 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.2 MiB/s wr, 10 op/s; 5.9 MiB/s, 3 objects/s recovering 2024-09-14T03:42:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:38 smithi195 ceph-mon[82388]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:42:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:38 smithi195 ceph-mon[82388]: pgmap v169: 121 pgs: 6 peering, 1 active+undersized, 114 active+clean; 473 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.2 MiB/s wr, 10 op/s; 5.9 MiB/s, 3 objects/s recovering 2024-09-14T03:42:39.379 INFO:tasks.rados.rados.0.smithi110.stdout:1523: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:42:39.380 INFO:tasks.rados.rados.0.smithi110.stdout:1523: finishing copy_from to smithi11075562-33 2024-09-14T03:42:39.380 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 993 (ObjNum 460 snap 149 seq_num 460) dirty exists 2024-09-14T03:42:39.380 INFO:tasks.rados.rados.0.smithi110.stdout:1520: expect (ObjNum 449 snap 144 seq_num 449) 2024-09-14T03:42:39.584 INFO:tasks.rados.rados.0.smithi110.stdout:1524: finishing copy_from to smithi11075562-46 2024-09-14T03:42:39.584 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 678 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:42:39.584 INFO:tasks.rados.rados.0.smithi110.stdout:1516: expect (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:42:40.118 INFO:tasks.rados.rados.0.smithi110.stdout:1522: expect (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:42:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:39 smithi110 ceph-mon[88262]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/918 objects degraded (9.368%), 10 pgs degraded) 2024-09-14T03:42:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:39 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:42:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:39 smithi110 ceph-mon[88262]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:42:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:39 smithi110 ceph-mon[86423]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/918 objects degraded (9.368%), 10 pgs degraded) 2024-09-14T03:42:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:39 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:42:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:39 smithi110 ceph-mon[86423]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:42:40.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:39 smithi195 ceph-mon[82388]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/918 objects degraded (9.368%), 10 pgs degraded) 2024-09-14T03:42:40.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:39 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:42:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:39 smithi195 ceph-mon[82388]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:42:40.483 INFO:tasks.rados.rados.0.smithi110.stdout:1526: seq_num 485 ranges {3891200=532480,4423680=401408} 2024-09-14T03:42:40.507 INFO:tasks.rados.rados.0.smithi110.stdout:1526: writing smithi11075562-12 from 3891200 to 4423680 tid 1 2024-09-14T03:42:40.509 INFO:tasks.rados.rados.0.smithi110.stdout:1526: writing smithi11075562-12 from 4423680 to 4825088 tid 2 2024-09-14T03:42:40.509 INFO:tasks.rados.rados.0.smithi110.stdout:1516: done (6 left) 2024-09-14T03:42:40.509 INFO:tasks.rados.rados.0.smithi110.stdout:1520: done (5 left) 2024-09-14T03:42:40.509 INFO:tasks.rados.rados.0.smithi110.stdout:1521: done (4 left) 2024-09-14T03:42:40.509 INFO:tasks.rados.rados.0.smithi110.stdout:1522: done (3 left) 2024-09-14T03:42:40.509 INFO:tasks.rados.rados.0.smithi110.stdout:1523: done (2 left) 2024-09-14T03:42:40.510 INFO:tasks.rados.rados.0.smithi110.stdout:1524: done (1 left) 2024-09-14T03:42:40.510 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 154 2024-09-14T03:42:40.510 INFO:tasks.rados.rados.0.smithi110.stdout:1527: seq_num 486 ranges {12886016=548864,13434880=401408} 2024-09-14T03:42:40.579 INFO:tasks.rados.rados.0.smithi110.stdout:1527: writing smithi11075562-23 from 12886016 to 13434880 tid 1 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:1527: writing smithi11075562-23 from 13434880 to 13836288 tid 2 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:1528: copy_from oid 35 from oid 5 current snap is 154 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:1526: finishing write tid 1 to smithi11075562-12 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:1526: finishing write tid 2 to smithi11075562-12 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:1526: finishing write tid 3 to smithi11075562-12 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:1526: finishing write tid 4 to smithi11075562-12 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1034 (ObjNum 485 snap 154 seq_num 485) dirty exists 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:1526: left oid 12 (ObjNum 485 snap 154 seq_num 485) 2024-09-14T03:42:40.582 INFO:tasks.rados.rados.0.smithi110.stdout:1526: done (2 left) 2024-09-14T03:42:40.583 INFO:tasks.rados.rados.0.smithi110.stdout:1529: read oid 14 snap -1 2024-09-14T03:42:40.583 INFO:tasks.rados.rados.0.smithi110.stdout:1529: expect (ObjNum 443 snap 142 seq_num 443) 2024-09-14T03:42:40.583 INFO:tasks.rados.rados.0.smithi110.stdout:1530: snap_remove snap 126 2024-09-14T03:42:40.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:40.741 INFO:tasks.rados.rados.0.smithi110.stdout:1530: done (3 left) 2024-09-14T03:42:40.741 INFO:tasks.rados.rados.0.smithi110.stdout:1531: snap_remove snap 150 2024-09-14T03:42:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:40 smithi110 ceph-mon[88262]: pgmap v171: 121 pgs: 121 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 4.5 MiB/s wr, 16 op/s; 12 MiB/s, 4 objects/s recovering 2024-09-14T03:42:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:40 smithi110 ceph-mon[86423]: pgmap v171: 121 pgs: 121 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 4.5 MiB/s wr, 16 op/s; 12 MiB/s, 4 objects/s recovering 2024-09-14T03:42:41.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:40 smithi195 ceph-mon[82388]: pgmap v171: 121 pgs: 121 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 4.5 MiB/s wr, 16 op/s; 12 MiB/s, 4 objects/s recovering 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1531: done (3 left) 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1532: delete oid 47 current snap is 154 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1527: finishing write tid 1 to smithi11075562-23 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1527: finishing write tid 2 to smithi11075562-23 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1527: finishing write tid 3 to smithi11075562-23 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1527: finishing write tid 4 to smithi11075562-23 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 527 (ObjNum 486 snap 154 seq_num 486) dirty exists 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1527: left oid 23 (ObjNum 486 snap 154 seq_num 486) 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1528: finishing copy_from racing read to smithi11075562-35 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:1528: finishing copy_from to smithi11075562-35 2024-09-14T03:42:41.747 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1012 (ObjNum 471 snap 151 seq_num 471) dirty exists 2024-09-14T03:42:41.748 INFO:tasks.rados.rados.0.smithi110.stdout:1529: expect (ObjNum 443 snap 142 seq_num 443) 2024-09-14T03:42:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:41 smithi110 ceph-mon[88262]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:42:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:42:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:42:41] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.51.0" 2024-09-14T03:42:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:41 smithi110 ceph-mon[86423]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:42:41.968 INFO:tasks.rados.rados.0.smithi110.stdout:1527: done (3 left) 2024-09-14T03:42:41.968 INFO:tasks.rados.rados.0.smithi110.stdout:1528: done (2 left) 2024-09-14T03:42:41.968 INFO:tasks.rados.rados.0.smithi110.stdout:1529: done (1 left) 2024-09-14T03:42:41.968 INFO:tasks.rados.rados.0.smithi110.stdout:1532: done (0 left) 2024-09-14T03:42:41.968 INFO:tasks.rados.rados.0.smithi110.stdout:1533: snap_remove snap 153 2024-09-14T03:42:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:41 smithi195 ceph-mon[82388]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:42:42.751 INFO:tasks.rados.rados.0.smithi110.stdout:1533: done (0 left) 2024-09-14T03:42:42.752 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 154 2024-09-14T03:42:42.752 INFO:tasks.rados.rados.0.smithi110.stdout:1534: seq_num 488 ranges {1671168=581632,2252800=401408} 2024-09-14T03:42:42.764 INFO:tasks.rados.rados.0.smithi110.stdout:1534: writing smithi11075562-16 from 1671168 to 2252800 tid 1 2024-09-14T03:42:42.766 INFO:tasks.rados.rados.0.smithi110.stdout:1534: writing smithi11075562-16 from 2252800 to 2654208 tid 2 2024-09-14T03:42:42.766 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 154 2024-09-14T03:42:42.766 INFO:tasks.rados.rados.0.smithi110.stdout:1535: seq_num 489 ranges {0=598016,598016=401408} 2024-09-14T03:42:42.770 INFO:tasks.rados.rados.0.smithi110.stdout:1535: writing smithi11075562-15 from 0 to 598016 tid 1 2024-09-14T03:42:42.772 INFO:tasks.rados.rados.0.smithi110.stdout:1535: writing smithi11075562-15 from 598016 to 999424 tid 2 2024-09-14T03:42:42.772 INFO:tasks.rados.rados.0.smithi110.stdout:1536: snap_remove snap 152 2024-09-14T03:42:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:42 smithi195 ceph-mon[82388]: pgmap v173: 121 pgs: 121 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.6 MiB/s wr, 13 op/s; 9.6 MiB/s, 3 objects/s recovering 2024-09-14T03:42:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:42 smithi195 ceph-mon[82388]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:42:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:42 smithi110 ceph-mon[88262]: pgmap v173: 121 pgs: 121 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.6 MiB/s wr, 13 op/s; 9.6 MiB/s, 3 objects/s recovering 2024-09-14T03:42:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:42 smithi110 ceph-mon[88262]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:42:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:42 smithi110 ceph-mon[86423]: pgmap v173: 121 pgs: 121 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.6 MiB/s wr, 13 op/s; 9.6 MiB/s, 3 objects/s recovering 2024-09-14T03:42:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:42 smithi110 ceph-mon[86423]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:42:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:43.688 INFO:tasks.rados.rados.0.smithi110.stdout:1536: done (2 left) 2024-09-14T03:42:43.688 INFO:tasks.rados.rados.0.smithi110.stdout:1537: read oid 20 snap -1 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1537: expect (ObjNum 445 snap 143 seq_num 445) 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1538: delete oid 50 current snap is 154 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1534: finishing write tid 1 to smithi11075562-16 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1534: finishing write tid 2 to smithi11075562-16 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1534: finishing write tid 3 to smithi11075562-16 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1534: finishing write tid 4 to smithi11075562-16 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 811 (ObjNum 488 snap 154 seq_num 488) dirty exists 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1534: left oid 16 (ObjNum 488 snap 154 seq_num 488) 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1535: finishing write tid 1 to smithi11075562-15 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1535: finishing write tid 2 to smithi11075562-15 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1535: finishing write tid 3 to smithi11075562-15 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:1535: finishing write tid 4 to smithi11075562-15 2024-09-14T03:42:43.689 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 932 (ObjNum 489 snap 154 seq_num 489) dirty exists 2024-09-14T03:42:43.690 INFO:tasks.rados.rados.0.smithi110.stdout:1535: left oid 15 (ObjNum 489 snap 154 seq_num 489) 2024-09-14T03:42:43.697 INFO:tasks.rados.rados.0.smithi110.stdout:1534: done (3 left) 2024-09-14T03:42:43.697 INFO:tasks.rados.rados.0.smithi110.stdout:1535: done (2 left) 2024-09-14T03:42:43.697 INFO:tasks.rados.rados.0.smithi110.stdout:1538: done (1 left) 2024-09-14T03:42:43.698 INFO:tasks.rados.rados.0.smithi110.stdout:1539: read oid 36 snap 140 2024-09-14T03:42:43.698 INFO:tasks.rados.rados.0.smithi110.stdout:1539: expect deleted 2024-09-14T03:42:43.698 INFO:tasks.rados.rados.0.smithi110.stdout:1540: rollback oid 29 current snap is 154 2024-09-14T03:42:43.698 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 29 to 140 2024-09-14T03:42:43.698 INFO:tasks.rados.rados.0.smithi110.stdout:1541: snap_create 2024-09-14T03:42:43.699 INFO:tasks.rados.rados.0.smithi110.stdout:1537: expect (ObjNum 445 snap 143 seq_num 445) 2024-09-14T03:42:43.713 INFO:tasks.rados.rados.0.smithi110.stdout:1540: finishing rollback tid 0 to smithi11075562-29 2024-09-14T03:42:43.713 INFO:tasks.rados.rados.0.smithi110.stdout:1540: finishing rollback tid 1 to smithi11075562-29 2024-09-14T03:42:43.713 INFO:tasks.rados.rados.0.smithi110.stdout:1540: finishing rollback tid 2 to smithi11075562-29 2024-09-14T03:42:43.713 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 495 (ObjNum 431 snap 138 seq_num 431) dirty exists 2024-09-14T03:42:44.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:43 smithi195 ceph-mon[82388]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:42:44.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:43 smithi195 ceph-mon[82388]: osdmap e367: 8 total, 8 up, 8 in 2024-09-14T03:42:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:43 smithi110 ceph-mon[88262]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:42:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:43 smithi110 ceph-mon[88262]: osdmap e367: 8 total, 8 up, 8 in 2024-09-14T03:42:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:43 smithi110 ceph-mon[86423]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:42:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:43 smithi110 ceph-mon[86423]: osdmap e367: 8 total, 8 up, 8 in 2024-09-14T03:42:44.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:44.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:42:44.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:44.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:44.695 INFO:tasks.rados.rados.0.smithi110.stdout:1537: done (3 left) 2024-09-14T03:42:44.695 INFO:tasks.rados.rados.0.smithi110.stdout:1539: done (2 left) 2024-09-14T03:42:44.695 INFO:tasks.rados.rados.0.smithi110.stdout:1540: done (1 left) 2024-09-14T03:42:44.695 INFO:tasks.rados.rados.0.smithi110.stdout:1541: done (0 left) 2024-09-14T03:42:44.696 INFO:tasks.rados.rados.0.smithi110.stdout:1542: rollback oid 29 current snap is 155 2024-09-14T03:42:44.696 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 29 to 154 2024-09-14T03:42:44.696 INFO:tasks.rados.rados.0.smithi110.stdout:1543: read oid 47 snap -1 2024-09-14T03:42:44.696 INFO:tasks.rados.rados.0.smithi110.stdout:1543: expect deleted 2024-09-14T03:42:44.696 INFO:tasks.rados.rados.0.smithi110.stdout:1544: snap_remove snap 139 2024-09-14T03:42:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:44 smithi110 ceph-mon[86423]: pgmap v177: 121 pgs: 121 active+clean; 502 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.9 MiB/s wr, 21 op/s 2024-09-14T03:42:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:44 smithi110 ceph-mon[86423]: osdmap e368: 8 total, 8 up, 8 in 2024-09-14T03:42:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:44 smithi110 ceph-mon[88262]: pgmap v177: 121 pgs: 121 active+clean; 502 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.9 MiB/s wr, 21 op/s 2024-09-14T03:42:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:44 smithi110 ceph-mon[88262]: osdmap e368: 8 total, 8 up, 8 in 2024-09-14T03:42:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:44 smithi195 ceph-mon[82388]: pgmap v177: 121 pgs: 121 active+clean; 502 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.9 MiB/s wr, 21 op/s 2024-09-14T03:42:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:44 smithi195 ceph-mon[82388]: osdmap e368: 8 total, 8 up, 8 in 2024-09-14T03:42:45.698 INFO:tasks.rados.rados.0.smithi110.stdout:1544: done (2 left) 2024-09-14T03:42:45.698 INFO:tasks.rados.rados.0.smithi110.stdout:1545: rollback oid 38 current snap is 155 2024-09-14T03:42:45.698 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 145 2024-09-14T03:42:45.699 INFO:tasks.rados.rados.0.smithi110.stdout:1542: finishing rollback tid 0 to smithi11075562-29 2024-09-14T03:42:45.699 INFO:tasks.rados.rados.0.smithi110.stdout:1542: finishing rollback tid 1 to smithi11075562-29 2024-09-14T03:42:45.699 INFO:tasks.rados.rados.0.smithi110.stdout:1542: finishing rollback tid 2 to smithi11075562-29 2024-09-14T03:42:45.699 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 498 (ObjNum 431 snap 138 seq_num 431) dirty exists 2024-09-14T03:42:45.699 INFO:tasks.rados.rados.0.smithi110.stdout:1542: done (2 left) 2024-09-14T03:42:45.699 INFO:tasks.rados.rados.0.smithi110.stdout:1543: done (1 left) 2024-09-14T03:42:45.699 INFO:tasks.rados.rados.0.smithi110.stdout:1546: snap_remove snap 147 2024-09-14T03:42:46.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:46.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:46.702 INFO:tasks.rados.rados.0.smithi110.stdout:1546: done (1 left) 2024-09-14T03:42:46.702 INFO:tasks.rados.rados.0.smithi110.stdout:1547: setattr oid 31 current snap is 155 2024-09-14T03:42:46.703 INFO:tasks.rados.rados.0.smithi110.stdout:1545: finishing rollback tid 0 to smithi11075562-38 2024-09-14T03:42:46.703 INFO:tasks.rados.rados.0.smithi110.stdout:1545: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:42:46.703 INFO:tasks.rados.rados.0.smithi110.stdout:1545: finishing rollback tid 2 to smithi11075562-38 2024-09-14T03:42:46.703 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 501 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:42:46.703 INFO:tasks.rados.rados.0.smithi110.stdout:1545: done (1 left) 2024-09-14T03:42:46.703 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 155 2024-09-14T03:42:46.703 INFO:tasks.rados.rados.0.smithi110.stdout:1548: seq_num 491 ranges {0=630784,630784=401408} 2024-09-14T03:42:46.707 INFO:tasks.rados.rados.0.smithi110.stdout:1548: writing smithi11075562-6 from 0 to 630784 tid 1 2024-09-14T03:42:46.710 INFO:tasks.rados.rados.0.smithi110.stdout:1548: writing smithi11075562-6 from 630784 to 1032192 tid 2 2024-09-14T03:42:46.710 INFO:tasks.rados.rados.0.smithi110.stdout:1549: snap_remove snap 144 2024-09-14T03:42:46.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:46 smithi195 ceph-mon[82388]: osdmap e369: 8 total, 8 up, 8 in 2024-09-14T03:42:46.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:46 smithi195 ceph-mon[82388]: pgmap v180: 121 pgs: 121 active+clean; 502 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.9 MiB/s wr, 21 op/s 2024-09-14T03:42:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:46 smithi110 ceph-mon[88262]: osdmap e369: 8 total, 8 up, 8 in 2024-09-14T03:42:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:46 smithi110 ceph-mon[88262]: pgmap v180: 121 pgs: 121 active+clean; 502 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.9 MiB/s wr, 21 op/s 2024-09-14T03:42:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:46 smithi110 ceph-mon[86423]: osdmap e369: 8 total, 8 up, 8 in 2024-09-14T03:42:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:46 smithi110 ceph-mon[86423]: pgmap v180: 121 pgs: 121 active+clean; 502 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.9 MiB/s wr, 21 op/s 2024-09-14T03:42:47.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:47.707 INFO:tasks.rados.rados.0.smithi110.stdout:1549: done (2 left) 2024-09-14T03:42:47.707 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 155 2024-09-14T03:42:47.707 INFO:tasks.rados.rados.0.smithi110.stdout:1550: seq_num 492 ranges {6365184=647168,7012352=401408} 2024-09-14T03:42:47.746 INFO:tasks.rados.rados.0.smithi110.stdout:1550: writing smithi11075562-46 from 6365184 to 7012352 tid 1 2024-09-14T03:42:47.748 INFO:tasks.rados.rados.0.smithi110.stdout:1550: writing smithi11075562-46 from 7012352 to 7413760 tid 2 2024-09-14T03:42:47.748 INFO:tasks.rados.rados.0.smithi110.stdout:1551: rollback oid 40 current snap is 155 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 40 to 142 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 508 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1548: finishing write tid 1 to smithi11075562-6 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1548: finishing write tid 2 to smithi11075562-6 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1548: finishing write tid 3 to smithi11075562-6 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1548: finishing write tid 4 to smithi11075562-6 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 828 (ObjNum 491 snap 155 seq_num 491) dirty exists 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1548: left oid 6 (ObjNum 491 snap 155 seq_num 491) 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1547: done (3 left) 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1548: done (2 left) 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1552: copy_from oid 28 from oid 31 current snap is 155 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 155 2024-09-14T03:42:47.749 INFO:tasks.rados.rados.0.smithi110.stdout:1553: seq_num 493 ranges {24576=663552,688128=401408} 2024-09-14T03:42:47.753 INFO:tasks.rados.rados.0.smithi110.stdout:1553: writing smithi11075562-29 from 24576 to 688128 tid 1 2024-09-14T03:42:47.755 INFO:tasks.rados.rados.0.smithi110.stdout:1553: writing smithi11075562-29 from 688128 to 1089536 tid 2 2024-09-14T03:42:47.755 INFO:tasks.rados.rados.0.smithi110.stdout:1554: read oid 3 snap -1 2024-09-14T03:42:47.755 INFO:tasks.rados.rados.0.smithi110.stdout:1554: expect deleted 2024-09-14T03:42:47.755 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 155 2024-09-14T03:42:47.756 INFO:tasks.rados.rados.0.smithi110.stdout:1550: finishing write tid 1 to smithi11075562-46 2024-09-14T03:42:47.756 INFO:tasks.rados.rados.0.smithi110.stdout:1555: seq_num 494 ranges {4259840=679936,4939776=401408} 2024-09-14T03:42:47.782 INFO:tasks.rados.rados.0.smithi110.stdout:1555: writing smithi11075562-14 from 4259840 to 4939776 tid 1 2024-09-14T03:42:47.785 INFO:tasks.rados.rados.0.smithi110.stdout:1555: writing smithi11075562-14 from 4939776 to 5341184 tid 2 2024-09-14T03:42:47.786 INFO:tasks.rados.rados.0.smithi110.stdout:1556: delete oid 34 current snap is 155 2024-09-14T03:42:47.786 INFO:tasks.rados.rados.0.smithi110.stdout:1552: finishing copy_from racing read to smithi11075562-28 2024-09-14T03:42:47.786 INFO:tasks.rados.rados.0.smithi110.stdout:1550: finishing write tid 2 to smithi11075562-46 2024-09-14T03:42:47.786 INFO:tasks.rados.rados.0.smithi110.stdout:1550: finishing write tid 3 to smithi11075562-46 2024-09-14T03:42:47.786 INFO:tasks.rados.rados.0.smithi110.stdout:1552: finishing copy_from to smithi11075562-28 2024-09-14T03:42:47.787 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 508 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:42:47.787 INFO:tasks.rados.rados.0.smithi110.stdout:1551: finishing rollback tid 0 to smithi11075562-40 2024-09-14T03:42:47.787 INFO:tasks.rados.rados.0.smithi110.stdout:1551: finishing rollback tid 1 to smithi11075562-40 2024-09-14T03:42:47.787 INFO:tasks.rados.rados.0.smithi110.stdout:1551: finishing rollback tid 2 to smithi11075562-40 2024-09-14T03:42:47.787 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 838 (ObjNum 441 snap 142 seq_num 441) dirty exists 2024-09-14T03:42:47.787 INFO:tasks.rados.rados.0.smithi110.stdout:1550: finishing write tid 4 to smithi11075562-46 2024-09-14T03:42:47.788 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 996 (ObjNum 492 snap 155 seq_num 492) dirty exists 2024-09-14T03:42:47.788 INFO:tasks.rados.rados.0.smithi110.stdout:1550: left oid 46 (ObjNum 492 snap 155 seq_num 492) 2024-09-14T03:42:47.788 INFO:tasks.rados.rados.0.smithi110.stdout:1553: finishing write tid 1 to smithi11075562-29 2024-09-14T03:42:47.788 INFO:tasks.rados.rados.0.smithi110.stdout:1553: finishing write tid 2 to smithi11075562-29 2024-09-14T03:42:47.788 INFO:tasks.rados.rados.0.smithi110.stdout:1553: finishing write tid 3 to smithi11075562-29 2024-09-14T03:42:47.789 INFO:tasks.rados.rados.0.smithi110.stdout:1553: finishing write tid 4 to smithi11075562-29 2024-09-14T03:42:47.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 504 (ObjNum 493 snap 155 seq_num 493) dirty exists 2024-09-14T03:42:47.789 INFO:tasks.rados.rados.0.smithi110.stdout:1553: left oid 29 (ObjNum 493 snap 155 seq_num 493) 2024-09-14T03:42:47.791 INFO:tasks.rados.rados.0.smithi110.stdout:1550: done (6 left) 2024-09-14T03:42:47.791 INFO:tasks.rados.rados.0.smithi110.stdout:1551: done (5 left) 2024-09-14T03:42:47.791 INFO:tasks.rados.rados.0.smithi110.stdout:1552: done (4 left) 2024-09-14T03:42:47.792 INFO:tasks.rados.rados.0.smithi110.stdout:1553: done (3 left) 2024-09-14T03:42:47.792 INFO:tasks.rados.rados.0.smithi110.stdout:1554: done (2 left) 2024-09-14T03:42:47.792 INFO:tasks.rados.rados.0.smithi110.stdout:1556: done (1 left) 2024-09-14T03:42:47.792 INFO:tasks.rados.rados.0.smithi110.stdout:1557: snap_create 2024-09-14T03:42:47.792 INFO:tasks.rados.rados.0.smithi110.stdout:1555: finishing write tid 1 to smithi11075562-14 2024-09-14T03:42:47.792 INFO:tasks.rados.rados.0.smithi110.stdout:1555: finishing write tid 2 to smithi11075562-14 2024-09-14T03:42:47.793 INFO:tasks.rados.rados.0.smithi110.stdout:1555: finishing write tid 3 to smithi11075562-14 2024-09-14T03:42:47.793 INFO:tasks.rados.rados.0.smithi110.stdout:1555: finishing write tid 4 to smithi11075562-14 2024-09-14T03:42:47.793 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 859 (ObjNum 494 snap 155 seq_num 494) dirty exists 2024-09-14T03:42:47.793 INFO:tasks.rados.rados.0.smithi110.stdout:1555: left oid 14 (ObjNum 494 snap 155 seq_num 494) 2024-09-14T03:42:47.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:47 smithi195 ceph-mon[82388]: osdmap e370: 8 total, 8 up, 8 in 2024-09-14T03:42:47.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:47 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:47 smithi110 ceph-mon[88262]: osdmap e370: 8 total, 8 up, 8 in 2024-09-14T03:42:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:47 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:47 smithi110 ceph-mon[86423]: osdmap e370: 8 total, 8 up, 8 in 2024-09-14T03:42:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:47 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:48.690 INFO:tasks.rados.rados.0.smithi110.stdout:1555: done (1 left) 2024-09-14T03:42:48.690 INFO:tasks.rados.rados.0.smithi110.stdout:1557: done (0 left) 2024-09-14T03:42:48.690 INFO:tasks.rados.rados.0.smithi110.stdout:1558: rollback oid 30 current snap is 156 2024-09-14T03:42:48.690 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 129 2024-09-14T03:42:48.690 INFO:tasks.rados.rados.0.smithi110.stdout:1559: snap_remove snap 107 2024-09-14T03:42:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:48 smithi195 ceph-mon[82388]: osdmap e371: 8 total, 8 up, 8 in 2024-09-14T03:42:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:48 smithi195 ceph-mon[82388]: pgmap v183: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 503 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 268 KiB/s rd, 244 KiB/s wr, 8 op/s 2024-09-14T03:42:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:48 smithi195 ceph-mon[82388]: osdmap e372: 8 total, 8 up, 8 in 2024-09-14T03:42:49.142 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:48 smithi110 ceph-mon[88262]: osdmap e371: 8 total, 8 up, 8 in 2024-09-14T03:42:49.142 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:48 smithi110 ceph-mon[88262]: pgmap v183: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 503 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 268 KiB/s rd, 244 KiB/s wr, 8 op/s 2024-09-14T03:42:49.143 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:48 smithi110 ceph-mon[88262]: osdmap e372: 8 total, 8 up, 8 in 2024-09-14T03:42:49.143 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:48 smithi110 ceph-mon[86423]: osdmap e371: 8 total, 8 up, 8 in 2024-09-14T03:42:49.143 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:48 smithi110 ceph-mon[86423]: pgmap v183: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 503 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 268 KiB/s rd, 244 KiB/s wr, 8 op/s 2024-09-14T03:42:49.143 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:48 smithi110 ceph-mon[86423]: osdmap e372: 8 total, 8 up, 8 in 2024-09-14T03:42:49.691 INFO:tasks.rados.rados.0.smithi110.stdout:1559: done (1 left) 2024-09-14T03:42:49.691 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 156 2024-09-14T03:42:49.691 INFO:tasks.rados.rados.0.smithi110.stdout:1558: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:42:49.692 INFO:tasks.rados.rados.0.smithi110.stdout:1558: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:42:49.692 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 942 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:42:49.692 INFO:tasks.rados.rados.0.smithi110.stdout:1560: seq_num 496 ranges {352256=712704,1064960=401408} 2024-09-14T03:42:49.698 INFO:tasks.rados.rados.0.smithi110.stdout:1560: writing smithi11075562-20 from 352256 to 1064960 tid 1 2024-09-14T03:42:49.701 INFO:tasks.rados.rados.0.smithi110.stdout:1560: writing smithi11075562-20 from 1064960 to 1466368 tid 2 2024-09-14T03:42:49.701 INFO:tasks.rados.rados.0.smithi110.stdout:1558: done (1 left) 2024-09-14T03:42:49.701 INFO:tasks.rados.rados.0.smithi110.stdout:1561: delete oid 36 current snap is 156 2024-09-14T03:42:49.709 INFO:tasks.rados.rados.0.smithi110.stdout:1560: finishing write tid 1 to smithi11075562-20 2024-09-14T03:42:49.710 INFO:tasks.rados.rados.0.smithi110.stdout:1561: done (1 left) 2024-09-14T03:42:49.710 INFO:tasks.rados.rados.0.smithi110.stdout:1562: snap_remove snap 67 2024-09-14T03:42:50.163 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:49 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:42:50.163 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:49 smithi110 ceph-mon[88262]: osdmap e373: 8 total, 8 up, 8 in 2024-09-14T03:42:50.164 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:49 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:42:50.164 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:49 smithi110 ceph-mon[86423]: osdmap e373: 8 total, 8 up, 8 in 2024-09-14T03:42:50.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:49 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:42:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:49 smithi195 ceph-mon[82388]: osdmap e373: 8 total, 8 up, 8 in 2024-09-14T03:42:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:50.717 INFO:tasks.rados.rados.0.smithi110.stdout:1562: done (1 left) 2024-09-14T03:42:50.717 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 156 2024-09-14T03:42:50.717 INFO:tasks.rados.rados.0.smithi110.stdout:1560: finishing write tid 2 to smithi11075562-20 2024-09-14T03:42:50.717 INFO:tasks.rados.rados.0.smithi110.stdout:1563: seq_num 498 ranges {6365184=745472,7110656=401408} 2024-09-14T03:42:50.758 INFO:tasks.rados.rados.0.smithi110.stdout:1563: writing smithi11075562-25 from 6365184 to 7110656 tid 1 2024-09-14T03:42:50.760 INFO:tasks.rados.rados.0.smithi110.stdout:1563: writing smithi11075562-25 from 7110656 to 7512064 tid 2 2024-09-14T03:42:50.760 INFO:tasks.rados.rados.0.smithi110.stdout:1564: read oid 22 snap -1 2024-09-14T03:42:50.761 INFO:tasks.rados.rados.0.smithi110.stdout:1564: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:42:50.761 INFO:tasks.rados.rados.0.smithi110.stdout:1565: snap_remove snap 132 2024-09-14T03:42:50.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:42:50.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[88262]: Upgrade: osd.2 is safe to restart 2024-09-14T03:42:50.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[88262]: pgmap v186: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 505 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 268 KiB/s rd, 736 KiB/s wr, 11 op/s 2024-09-14T03:42:50.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:50.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:42:50.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:50.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:42:50.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[86423]: Upgrade: osd.2 is safe to restart 2024-09-14T03:42:50.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[86423]: pgmap v186: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 505 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 268 KiB/s rd, 736 KiB/s wr, 11 op/s 2024-09-14T03:42:50.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:50.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:42:50.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:50 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:50 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:42:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:50 smithi195 ceph-mon[82388]: Upgrade: osd.2 is safe to restart 2024-09-14T03:42:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:50 smithi195 ceph-mon[82388]: pgmap v186: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 505 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 268 KiB/s rd, 736 KiB/s wr, 11 op/s 2024-09-14T03:42:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:50 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:50 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:42:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:50 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:42:51.722 INFO:tasks.rados.rados.0.smithi110.stdout:1565: done (3 left) 2024-09-14T03:42:51.722 INFO:tasks.rados.rados.0.smithi110.stdout:1566: read oid 43 snap -1 2024-09-14T03:42:51.722 INFO:tasks.rados.rados.0.smithi110.stdout:1566: expect (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:42:51.722 INFO:tasks.rados.rados.0.smithi110.stdout:1567: rmattr oid 8 current snap is 156 2024-09-14T03:42:51.722 INFO:tasks.rados.rados.0.smithi110.stdout:1567: done (4 left) 2024-09-14T03:42:51.722 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 156 2024-09-14T03:42:51.722 INFO:tasks.rados.rados.0.smithi110.stdout:1568: seq_num 499 ranges {581632=761856,1343488=401408} 2024-09-14T03:42:51.732 INFO:tasks.rados.rados.0.smithi110.stdout:1568: writing smithi11075562-38 from 581632 to 1343488 tid 1 2024-09-14T03:42:51.733 INFO:tasks.rados.rados.0.smithi110.stdout:1568: writing smithi11075562-38 from 1343488 to 1744896 tid 2 2024-09-14T03:42:51.734 INFO:tasks.rados.rados.0.smithi110.stdout:1560: finishing write tid 3 to smithi11075562-20 2024-09-14T03:42:51.734 INFO:tasks.rados.rados.0.smithi110.stdout:1560: finishing write tid 4 to smithi11075562-20 2024-09-14T03:42:51.734 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 999 (ObjNum 496 snap 156 seq_num 496) dirty exists 2024-09-14T03:42:51.734 INFO:tasks.rados.rados.0.smithi110.stdout:1560: left oid 20 (ObjNum 496 snap 156 seq_num 496) 2024-09-14T03:42:51.734 INFO:tasks.rados.rados.0.smithi110.stdout:1563: finishing write tid 1 to smithi11075562-25 2024-09-14T03:42:51.734 INFO:tasks.rados.rados.0.smithi110.stdout:1563: finishing write tid 2 to smithi11075562-25 2024-09-14T03:42:51.734 INFO:tasks.rados.rados.0.smithi110.stdout:1563: finishing write tid 3 to smithi11075562-25 2024-09-14T03:42:51.734 INFO:tasks.rados.rados.0.smithi110.stdout:1563: finishing write tid 4 to smithi11075562-25 2024-09-14T03:42:51.735 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1104 (ObjNum 498 snap 156 seq_num 498) dirty exists 2024-09-14T03:42:51.735 INFO:tasks.rados.rados.0.smithi110.stdout:1563: left oid 25 (ObjNum 498 snap 156 seq_num 498) 2024-09-14T03:42:51.735 INFO:tasks.rados.rados.0.smithi110.stdout:1564: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:42:51.819 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:42:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:42:51] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.51.0" 2024-09-14T03:42:51.819 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:51 smithi110 ceph-mon[86423]: Upgrade: Updating osd.2 2024-09-14T03:42:51.819 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:51 smithi110 ceph-mon[86423]: Deploying daemon osd.2 on smithi110 2024-09-14T03:42:51.819 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:51 smithi110 ceph-mon[86423]: osdmap e374: 8 total, 8 up, 8 in 2024-09-14T03:42:51.820 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:51 smithi110 ceph-mon[88262]: Upgrade: Updating osd.2 2024-09-14T03:42:51.820 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:51 smithi110 ceph-mon[88262]: Deploying daemon osd.2 on smithi110 2024-09-14T03:42:51.820 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:51 smithi110 ceph-mon[88262]: osdmap e374: 8 total, 8 up, 8 in 2024-09-14T03:42:52.058 INFO:tasks.rados.rados.0.smithi110.stdout:1568: finishing write tid 1 to smithi11075562-38 2024-09-14T03:42:52.058 INFO:tasks.rados.rados.0.smithi110.stdout:1568: finishing write tid 2 to smithi11075562-38 2024-09-14T03:42:52.058 INFO:tasks.rados.rados.0.smithi110.stdout:1568: finishing write tid 3 to smithi11075562-38 2024-09-14T03:42:52.058 INFO:tasks.rados.rados.0.smithi110.stdout:1568: finishing write tid 4 to smithi11075562-38 2024-09-14T03:42:52.058 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 507 (ObjNum 499 snap 156 seq_num 499) dirty exists 2024-09-14T03:42:52.059 INFO:tasks.rados.rados.0.smithi110.stdout:1568: left oid 38 (ObjNum 499 snap 156 seq_num 499) 2024-09-14T03:42:52.059 INFO:tasks.rados.rados.0.smithi110.stdout:1566: expect (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:42:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:51 smithi195 ceph-mon[82388]: Upgrade: Updating osd.2 2024-09-14T03:42:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:51 smithi195 ceph-mon[82388]: Deploying daemon osd.2 on smithi110 2024-09-14T03:42:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:51 smithi195 ceph-mon[82388]: osdmap e374: 8 total, 8 up, 8 in 2024-09-14T03:42:52.210 INFO:tasks.rados.rados.0.smithi110.stdout:1560: done (4 left) 2024-09-14T03:42:52.210 INFO:tasks.rados.rados.0.smithi110.stdout:1563: done (3 left) 2024-09-14T03:42:52.211 INFO:tasks.rados.rados.0.smithi110.stdout:1564: done (2 left) 2024-09-14T03:42:52.211 INFO:tasks.rados.rados.0.smithi110.stdout:1566: done (1 left) 2024-09-14T03:42:52.211 INFO:tasks.rados.rados.0.smithi110.stdout:1568: done (0 left) 2024-09-14T03:42:52.211 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 156 2024-09-14T03:42:52.211 INFO:tasks.rados.rados.0.smithi110.stdout:1569: seq_num 500 ranges {7520256=778240,8298496=401408} 2024-09-14T03:42:52.255 INFO:tasks.rados.rados.0.smithi110.stdout:1569: writing smithi11075562-10 from 7520256 to 8298496 tid 1 2024-09-14T03:42:52.257 INFO:tasks.rados.rados.0.smithi110.stdout:1569: writing smithi11075562-10 from 8298496 to 8699904 tid 2 2024-09-14T03:42:52.258 INFO:tasks.rados.rados.0.smithi110.stdout:1570: snap_create 2024-09-14T03:42:52.261 INFO:tasks.rados.rados.0.smithi110.stdout:1569: finishing write tid 1 to smithi11075562-10 2024-09-14T03:42:52.261 INFO:tasks.rados.rados.0.smithi110.stdout:1569: finishing write tid 2 to smithi11075562-10 2024-09-14T03:42:52.262 INFO:tasks.rados.rados.0.smithi110.stdout:1569: finishing write tid 3 to smithi11075562-10 2024-09-14T03:42:52.263 INFO:tasks.rados.rados.0.smithi110.stdout:1569: finishing write tid 4 to smithi11075562-10 2024-09-14T03:42:52.263 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 554 (ObjNum 500 snap 156 seq_num 500) dirty exists 2024-09-14T03:42:52.263 INFO:tasks.rados.rados.0.smithi110.stdout:1569: left oid 10 (ObjNum 500 snap 156 seq_num 500) 2024-09-14T03:42:52.429 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:42:52.728 INFO:tasks.rados.rados.0.smithi110.stdout:1569: done (1 left) 2024-09-14T03:42:52.728 INFO:tasks.rados.rados.0.smithi110.stdout:1570: done (0 left) 2024-09-14T03:42:52.728 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 157 2024-09-14T03:42:52.728 INFO:tasks.rados.rados.0.smithi110.stdout:1571: seq_num 501 ranges {876544=794624,1671168=401408} 2024-09-14T03:42:52.737 INFO:tasks.rados.rados.0.smithi110.stdout:1571: writing smithi11075562-9 from 876544 to 1671168 tid 1 2024-09-14T03:42:52.739 INFO:tasks.rados.rados.0.smithi110.stdout:1571: writing smithi11075562-9 from 1671168 to 2072576 tid 2 2024-09-14T03:42:52.739 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 157 2024-09-14T03:42:52.739 INFO:tasks.rados.rados.0.smithi110.stdout:1572: seq_num 502 ranges {8699904=409600,9109504=417792,9527296=385024} 2024-09-14T03:42:52.789 INFO:tasks.rados.rados.0.smithi110.stdout:1572: writing smithi11075562-10 from 8699904 to 9109504 tid 1 2024-09-14T03:42:52.791 INFO:tasks.rados.rados.0.smithi110.stdout:1572: writing smithi11075562-10 from 9109504 to 9527296 tid 2 2024-09-14T03:42:52.795 INFO:tasks.rados.rados.0.smithi110.stdout:1572: writing smithi11075562-10 from 9527296 to 9912320 tid 3 2024-09-14T03:42:52.796 INFO:tasks.rados.rados.0.smithi110.stdout:1573: copy_from oid 6 from oid 7 current snap is 157 2024-09-14T03:42:52.796 INFO:tasks.rados.rados.0.smithi110.stdout:1571: finishing write tid 1 to smithi11075562-9 2024-09-14T03:42:52.796 INFO:tasks.rados.rados.0.smithi110.stdout:1571: finishing write tid 2 to smithi11075562-9 2024-09-14T03:42:52.796 INFO:tasks.rados.rados.0.smithi110.stdout:1571: finishing write tid 3 to smithi11075562-9 2024-09-14T03:42:52.796 INFO:tasks.rados.rados.0.smithi110.stdout:1571: finishing write tid 4 to smithi11075562-9 2024-09-14T03:42:52.796 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1002 (ObjNum 501 snap 157 seq_num 501) dirty exists 2024-09-14T03:42:52.797 INFO:tasks.rados.rados.0.smithi110.stdout:1571: left oid 9 (ObjNum 501 snap 157 seq_num 501) 2024-09-14T03:42:52.797 INFO:tasks.rados.rados.0.smithi110.stdout:1571: done (2 left) 2024-09-14T03:42:52.797 INFO:tasks.rados.rados.0.smithi110.stdout:1574: rmattr oid 50 current snap is 157 2024-09-14T03:42:52.797 INFO:tasks.rados.rados.0.smithi110.stdout:1574: done (2 left) 2024-09-14T03:42:52.797 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 157 2024-09-14T03:42:52.797 INFO:tasks.rados.rados.0.smithi110.stdout:1575: seq_num 503 ranges {1744896=425984,2170880=704512,2875392=98304} 2024-09-14T03:42:52.807 INFO:tasks.rados.rados.0.smithi110.stdout:1575: writing smithi11075562-38 from 1744896 to 2170880 tid 1 2024-09-14T03:42:52.811 INFO:tasks.rados.rados.0.smithi110.stdout:1575: writing smithi11075562-38 from 2170880 to 2875392 tid 2 2024-09-14T03:42:52.812 INFO:tasks.rados.rados.0.smithi110.stdout:1575: writing smithi11075562-38 from 2875392 to 2973696 tid 3 2024-09-14T03:42:52.812 INFO:tasks.rados.rados.0.smithi110.stdout:1576: delete oid 18 current snap is 157 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:1572: finishing write tid 1 to smithi11075562-10 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:1572: finishing write tid 2 to smithi11075562-10 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:1572: finishing write tid 3 to smithi11075562-10 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:1572: finishing write tid 4 to smithi11075562-10 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:1572: finishing write tid 5 to smithi11075562-10 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 559 (ObjNum 502 snap 157 seq_num 502) dirty exists 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:1572: left oid 10 (ObjNum 502 snap 157 seq_num 502) 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:1573: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:42:52.813 INFO:tasks.rados.rados.0.smithi110.stdout:1573: finishing copy_from to smithi11075562-6 2024-09-14T03:42:52.814 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 487 (ObjNum 470 snap 150 seq_num 470) dirty exists 2024-09-14T03:42:52.814 INFO:tasks.rados.rados.0.smithi110.stdout:1575: finishing write tid 1 to smithi11075562-38 2024-09-14T03:42:52.814 INFO:tasks.rados.rados.0.smithi110.stdout:1572: done (3 left) 2024-09-14T03:42:52.815 INFO:tasks.rados.rados.0.smithi110.stdout:1573: done (2 left) 2024-09-14T03:42:52.815 INFO:tasks.rados.rados.0.smithi110.stdout:1576: done (1 left) 2024-09-14T03:42:52.815 INFO:tasks.rados.rados.0.smithi110.stdout:1577: read oid 11 snap -1 2024-09-14T03:42:52.815 INFO:tasks.rados.rados.0.smithi110.stdout:1577: expect (ObjNum 473 snap 152 seq_num 473) 2024-09-14T03:42:52.815 INFO:tasks.rados.rados.0.smithi110.stdout:1578: snap_create 2024-09-14T03:42:52.815 INFO:tasks.rados.rados.0.smithi110.stdout:1575: finishing write tid 2 to smithi11075562-38 2024-09-14T03:42:52.815 INFO:tasks.rados.rados.0.smithi110.stdout:1575: finishing write tid 3 to smithi11075562-38 2024-09-14T03:42:52.816 INFO:tasks.rados.rados.0.smithi110.stdout:1575: finishing write tid 4 to smithi11075562-38 2024-09-14T03:42:52.818 INFO:tasks.rados.rados.0.smithi110.stdout:1575: finishing write tid 5 to smithi11075562-38 2024-09-14T03:42:52.818 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 511 (ObjNum 503 snap 157 seq_num 503) dirty exists 2024-09-14T03:42:52.818 INFO:tasks.rados.rados.0.smithi110.stdout:1575: left oid 38 (ObjNum 503 snap 157 seq_num 503) 2024-09-14T03:42:52.825 INFO:tasks.rados.rados.0.smithi110.stdout:1577: expect (ObjNum 473 snap 152 seq_num 473) 2024-09-14T03:42:53.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:52 smithi110 ceph-mon[88262]: osdmap e375: 8 total, 8 up, 8 in 2024-09-14T03:42:53.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:52 smithi110 ceph-mon[88262]: pgmap v189: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 505 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 492 KiB/s wr, 2 op/s 2024-09-14T03:42:53.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:52 smithi110 ceph-mon[86423]: osdmap e375: 8 total, 8 up, 8 in 2024-09-14T03:42:53.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:52 smithi110 ceph-mon[86423]: pgmap v189: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 505 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 492 KiB/s wr, 2 op/s 2024-09-14T03:42:53.026 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:42:53.026 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (3m) 23s ago 7m 16.6M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:42:53.027 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (3m) 94s ago 7m 78.7M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:42:53.027 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (9m) 23s ago 9m 19.9M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:42:53.027 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (9m) 94s ago 9m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:42:53.027 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (2m) 94s ago 12m 452M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:42:53.027 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (3m) 23s ago 13m 523M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:42:53.027 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (2m) 23s ago 14m 51.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:42:53.028 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (99s) 94s ago 12m 21.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:42:53.028 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (117s) 23s ago 12m 42.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:42:53.028 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (3m) 23s ago 7m 9.80M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:42:53.028 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (3m) 94s ago 7m 9722k - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:42:53.028 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (60s) 23s ago 12m 149M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:42:53.028 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (27s) 23s ago 11m 11.9M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:42:53.029 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (11m) 23s ago 11m 176M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d9d6cb524364 2024-09-14T03:42:53.029 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (10m) 23s ago 10m 153M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:42:53.029 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (10m) 94s ago 10m 229M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:42:53.029 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (10m) 94s ago 10m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:42:53.029 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (9m) 94s ago 9m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:42:53.029 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (9m) 94s ago 9m 106M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:42:53.030 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (2m) 94s ago 8m 44.2M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:42:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:52 smithi195 ceph-mon[82388]: osdmap e375: 8 total, 8 up, 8 in 2024-09-14T03:42:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:52 smithi195 ceph-mon[82388]: pgmap v189: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 505 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 492 KiB/s wr, 2 op/s 2024-09-14T03:42:53.382 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:53 smithi110 systemd[1]: Stopping Ceph osd.2 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:42:53.393 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:42:53.393 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:42:53.393 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:42:53.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:42:53.393 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 6, 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8, 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:42:53.394 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "7/20 daemons upgraded", 2024-09-14T03:42:53.684 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:42:53.685 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:42:53.685 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:42:53.685 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[50653]: 2024-09-14T03:42:53.532+0000 7f525613e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:42:53.685 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[50653]: 2024-09-14T03:42:53.532+0000 7f525613e640 -1 osd.2 376 *** Got signal Terminated *** 2024-09-14T03:42:53.685 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[50653]: 2024-09-14T03:42:53.532+0000 7f525613e640 -1 osd.2 376 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:42:53.691 INFO:tasks.rados.rados.0.smithi110.stdout:1575: done (2 left) 2024-09-14T03:42:53.691 INFO:tasks.rados.rados.0.smithi110.stdout:1577: done (1 left) 2024-09-14T03:42:53.691 INFO:tasks.rados.rados.0.smithi110.stdout:1578: done (0 left) 2024-09-14T03:42:53.691 INFO:tasks.rados.rados.0.smithi110.stdout:1579: delete oid 24 current snap is 158 2024-09-14T03:42:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[86423]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[86423]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[86423]: osdmap e376: 8 total, 8 up, 8 in 2024-09-14T03:42:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1152790596' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[86423]: osd.2 marked itself down and dead 2024-09-14T03:42:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[86423]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:42:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[86423]: osdmap e377: 8 total, 7 up, 8 in 2024-09-14T03:42:53.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[88262]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:53.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[88262]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:53.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[88262]: osdmap e376: 8 total, 8 up, 8 in 2024-09-14T03:42:53.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1152790596' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:53.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[88262]: osd.2 marked itself down and dead 2024-09-14T03:42:53.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[88262]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:42:53.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:53 smithi110 ceph-mon[88262]: osdmap e377: 8 total, 7 up, 8 in 2024-09-14T03:42:54.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:53 smithi195 ceph-mon[82388]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:53 smithi195 ceph-mon[82388]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:53 smithi195 ceph-mon[82388]: osdmap e376: 8 total, 8 up, 8 in 2024-09-14T03:42:54.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:53 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1152790596' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:42:54.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:53 smithi195 ceph-mon[82388]: osd.2 marked itself down and dead 2024-09-14T03:42:54.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:53 smithi195 ceph-mon[82388]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:42:54.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:53 smithi195 ceph-mon[82388]: osdmap e377: 8 total, 7 up, 8 in 2024-09-14T03:42:54.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:54.517 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:54 smithi110 podman[101844]: 2024-09-14 03:42:54.23601691 +0000 UTC m=+0.827599049 container died d9d6cb524364ed970497ce8030d3e06d9d2f022d20bd70e66a57c73e4dd3a1b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=quincy-eb16b97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:42:54.705 INFO:tasks.rados.rados.0.smithi110.stdout:1579: done (0 left) 2024-09-14T03:42:54.705 INFO:tasks.rados.rados.0.smithi110.stdout:1580: snap_create 2024-09-14T03:42:54.884 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:54 smithi110 podman[101844]: 2024-09-14 03:42:54.623084069 +0000 UTC m=+1.214666215 container cleanup d9d6cb524364ed970497ce8030d3e06d9d2f022d20bd70e66a57c73e4dd3a1b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:42:54.884 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:54 smithi110 bash[101844]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2 2024-09-14T03:42:54.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:54 smithi110 ceph-mon[86423]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:54 smithi110 ceph-mon[86423]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:54 smithi110 ceph-mon[86423]: pgmap v192: 121 pgs: 14 stale+active+clean, 107 active+clean; 468 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.9 MiB/s wr, 10 op/s 2024-09-14T03:42:54.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:54 smithi110 ceph-mon[86423]: osdmap e378: 8 total, 7 up, 8 in 2024-09-14T03:42:54.885 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:54 smithi110 ceph-mon[88262]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.886 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:54 smithi110 ceph-mon[88262]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.886 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:54 smithi110 ceph-mon[88262]: pgmap v192: 121 pgs: 14 stale+active+clean, 107 active+clean; 468 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.9 MiB/s wr, 10 op/s 2024-09-14T03:42:54.886 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:54 smithi110 ceph-mon[88262]: osdmap e378: 8 total, 7 up, 8 in 2024-09-14T03:42:55.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:54 smithi110 podman[101883]: 2024-09-14 03:42:54.883777095 +0000 UTC m=+0.642119157 container remove d9d6cb524364ed970497ce8030d3e06d9d2f022d20bd70e66a57c73e4dd3a1b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, RELEASE=quincy-eb16b97, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:42:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:54 smithi195 ceph-mon[82388]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:54 smithi195 ceph-mon[82388]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:54 smithi195 ceph-mon[82388]: pgmap v192: 121 pgs: 14 stale+active+clean, 107 active+clean; 468 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.9 MiB/s wr, 10 op/s 2024-09-14T03:42:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:54 smithi195 ceph-mon[82388]: osdmap e378: 8 total, 7 up, 8 in 2024-09-14T03:42:55.505 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:55 smithi110 podman[101971]: 2024-09-14 03:42:55.234848534 +0000 UTC m=+0.104766995 container create ed94c9989a46c11a4c3811f648a0f254e485309554d5126d5bf51ae72a17745e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-14T03:42:55.505 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:55 smithi110 podman[101971]: 2024-09-14 03:42:55.145725796 +0000 UTC m=+0.015644215 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:42:55.505 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:55 smithi110 podman[101971]: 2024-09-14 03:42:55.504401474 +0000 UTC m=+0.374319893 container init ed94c9989a46c11a4c3811f648a0f254e485309554d5126d5bf51ae72a17745e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-14T03:42:55.693 INFO:tasks.rados.rados.0.smithi110.stdout:1580: done (0 left) 2024-09-14T03:42:55.694 INFO:tasks.rados.rados.0.smithi110.stdout:1581: snap_remove snap 151 2024-09-14T03:42:55.946 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:55 smithi110 podman[101971]: 2024-09-14 03:42:55.507884451 +0000 UTC m=+0.377802881 container start ed94c9989a46c11a4c3811f648a0f254e485309554d5126d5bf51ae72a17745e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-a278231, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T03:42:55.946 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:55 smithi110 podman[101971]: 2024-09-14 03:42:55.579043441 +0000 UTC m=+0.448961878 container attach ed94c9989a46c11a4c3811f648a0f254e485309554d5126d5bf51ae72a17745e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:42:55.947 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:55 smithi110 podman[101971]: 2024-09-14 03:42:55.698220338 +0000 UTC m=+0.568138764 container died ed94c9989a46c11a4c3811f648a0f254e485309554d5126d5bf51ae72a17745e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-14T03:42:56.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:56.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:56.696 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 podman[101994]: 2024-09-14 03:42:56.291810403 +0000 UTC m=+0.587497106 container remove ed94c9989a46c11a4c3811f648a0f254e485309554d5126d5bf51ae72a17745e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-a278231) 2024-09-14T03:42:56.696 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-14T03:42:56.697 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 systemd[1]: Stopped Ceph osd.2 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:42:56.697 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.2.service: Consumed 12.295s CPU time. 2024-09-14T03:42:56.697 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 systemd[1]: Starting Ceph osd.2 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:42:56.702 INFO:tasks.rados.rados.0.smithi110.stdout:1581: done (0 left) 2024-09-14T03:42:56.703 INFO:tasks.rados.rados.0.smithi110.stdout:1582: snap_create 2024-09-14T03:42:56.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:56 smithi195 ceph-mon[82388]: osdmap e379: 8 total, 7 up, 8 in 2024-09-14T03:42:56.950 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:56 smithi195 ceph-mon[82388]: pgmap v195: 121 pgs: 6 peering, 11 stale+active+clean, 104 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-14T03:42:56.987 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:56 smithi110 ceph-mon[88262]: osdmap e379: 8 total, 7 up, 8 in 2024-09-14T03:42:56.988 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:56 smithi110 ceph-mon[88262]: pgmap v195: 121 pgs: 6 peering, 11 stale+active+clean, 104 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-14T03:42:56.988 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:56 smithi110 ceph-mon[86423]: osdmap e379: 8 total, 7 up, 8 in 2024-09-14T03:42:56.988 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:56 smithi110 ceph-mon[86423]: pgmap v195: 121 pgs: 6 peering, 11 stale+active+clean, 104 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-14T03:42:56.988 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 podman[102095]: 2024-09-14 03:42:56.625769255 +0000 UTC m=+0.015841632 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:42:56.988 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 podman[102095]: 2024-09-14 03:42:56.734533985 +0000 UTC m=+0.124606408 container create 6ddad0a24533327a9c398ff60967bc96b9a93aeda1218633fd445d743f8c6109 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:42:57.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:42:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:42:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:42:57.448 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 podman[102095]: 2024-09-14 03:42:56.987198109 +0000 UTC m=+0.377270473 container init 6ddad0a24533327a9c398ff60967bc96b9a93aeda1218633fd445d743f8c6109 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:42:57.448 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:56 smithi110 podman[102095]: 2024-09-14 03:42:56.990567579 +0000 UTC m=+0.380639968 container start 6ddad0a24533327a9c398ff60967bc96b9a93aeda1218633fd445d743f8c6109 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-14T03:42:57.449 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 podman[102095]: 2024-09-14 03:42:57.070676877 +0000 UTC m=+0.460749241 container attach 6ddad0a24533327a9c398ff60967bc96b9a93aeda1218633fd445d743f8c6109 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T03:42:57.449 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:42:57.449 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:42:57.449 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:42:57.449 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:42:57.701 INFO:tasks.rados.rados.0.smithi110.stdout:1582: done (0 left) 2024-09-14T03:42:57.701 INFO:tasks.rados.rados.0.smithi110.stdout:1583: read oid 11 snap -1 2024-09-14T03:42:57.701 INFO:tasks.rados.rados.0.smithi110.stdout:1583: expect (ObjNum 473 snap 152 seq_num 473) 2024-09-14T03:42:57.701 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 160 2024-09-14T03:42:57.701 INFO:tasks.rados.rados.0.smithi110.stdout:1584: seq_num 506 ranges {0=483328,483328=761856,1245184=40960} 2024-09-14T03:42:57.704 INFO:tasks.rados.rados.0.smithi110.stdout:1584: writing smithi11075562-24 from 0 to 483328 tid 1 2024-09-14T03:42:57.709 INFO:tasks.rados.rados.0.smithi110.stdout:1584: writing smithi11075562-24 from 483328 to 1245184 tid 2 2024-09-14T03:42:57.709 INFO:tasks.rados.rados.0.smithi110.stdout:1584: writing smithi11075562-24 from 1245184 to 1286144 tid 3 2024-09-14T03:42:57.709 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 160 2024-09-14T03:42:57.710 INFO:tasks.rados.rados.0.smithi110.stdout:1585: seq_num 507 ranges {3620864=499712,4120576=638976,4759552=163840} 2024-09-14T03:42:57.732 INFO:tasks.rados.rados.0.smithi110.stdout:1585: writing smithi11075562-32 from 3620864 to 4120576 tid 1 2024-09-14T03:42:57.735 INFO:tasks.rados.rados.0.smithi110.stdout:1585: writing smithi11075562-32 from 4120576 to 4759552 tid 2 2024-09-14T03:42:57.737 INFO:tasks.rados.rados.0.smithi110.stdout:1585: writing smithi11075562-32 from 4759552 to 4923392 tid 3 2024-09-14T03:42:57.737 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 160 2024-09-14T03:42:57.737 INFO:tasks.rados.rados.0.smithi110.stdout:1584: finishing write tid 1 to smithi11075562-24 2024-09-14T03:42:57.737 INFO:tasks.rados.rados.0.smithi110.stdout:1586: seq_num 508 ranges {0=516096,516096=524288,1040384=278528} 2024-09-14T03:42:57.739 INFO:tasks.rados.rados.0.smithi110.stdout:1586: writing smithi11075562-36 from 0 to 516096 tid 1 2024-09-14T03:42:57.742 INFO:tasks.rados.rados.0.smithi110.stdout:1586: writing smithi11075562-36 from 516096 to 1040384 tid 2 2024-09-14T03:42:57.744 INFO:tasks.rados.rados.0.smithi110.stdout:1586: writing smithi11075562-36 from 1040384 to 1318912 tid 3 2024-09-14T03:42:57.744 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 160 2024-09-14T03:42:57.744 INFO:tasks.rados.rados.0.smithi110.stdout:1583: expect (ObjNum 473 snap 152 seq_num 473) 2024-09-14T03:42:57.821 INFO:tasks.rados.rados.0.smithi110.stdout:1584: finishing write tid 2 to smithi11075562-24 2024-09-14T03:42:57.821 INFO:tasks.rados.rados.0.smithi110.stdout:1584: finishing write tid 3 to smithi11075562-24 2024-09-14T03:42:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:1584: finishing write tid 4 to smithi11075562-24 2024-09-14T03:42:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:1584: finishing write tid 5 to smithi11075562-24 2024-09-14T03:42:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 565 (ObjNum 506 snap 160 seq_num 506) dirty exists 2024-09-14T03:42:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:1584: left oid 24 (ObjNum 506 snap 160 seq_num 506) 2024-09-14T03:42:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:1585: finishing write tid 1 to smithi11075562-32 2024-09-14T03:42:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:1587: seq_num 509 ranges {6365184=532480,6897664=409600,7307264=393216} 2024-09-14T03:42:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:1587: writing smithi11075562-22 from 6365184 to 6897664 tid 1 2024-09-14T03:42:57.859 INFO:tasks.rados.rados.0.smithi110.stdout:1587: writing smithi11075562-22 from 6897664 to 7307264 tid 2 2024-09-14T03:42:57.862 INFO:tasks.rados.rados.0.smithi110.stdout:1587: writing smithi11075562-22 from 7307264 to 7700480 tid 3 2024-09-14T03:42:57.862 INFO:tasks.rados.rados.0.smithi110.stdout:1583: done (4 left) 2024-09-14T03:42:57.862 INFO:tasks.rados.rados.0.smithi110.stdout:1584: done (3 left) 2024-09-14T03:42:57.862 INFO:tasks.rados.rados.0.smithi110.stdout:1588: snap_remove snap 136 2024-09-14T03:42:57.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-mon[86423]: osdmap e380: 8 total, 7 up, 8 in 2024-09-14T03:42:57.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:57.874 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:57.874 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-mon[88262]: osdmap e380: 8 total, 7 up, 8 in 2024-09-14T03:42:57.874 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:57.874 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:57.874 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:42:57.875 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:42:57.875 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T03:42:57.875 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T03:42:57.875 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T03:42:57.875 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T03:42:57.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:57 smithi195 ceph-mon[82388]: osdmap e380: 8 total, 7 up, 8 in 2024-09-14T03:42:57.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:57.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:42:58.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T03:42:58.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T03:42:58.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T03:42:58.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T03:42:58.197 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:42:58.197 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:42:58.197 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate[102111]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T03:42:58.197 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 bash[102095]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T03:42:58.197 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:57 smithi110 podman[102095]: 2024-09-14 03:42:57.901046609 +0000 UTC m=+1.291118973 container died 6ddad0a24533327a9c398ff60967bc96b9a93aeda1218633fd445d743f8c6109 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-14T03:42:58.668 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:58 smithi110 podman[102236]: 2024-09-14 03:42:58.46682844 +0000 UTC m=+0.555729411 container remove 6ddad0a24533327a9c398ff60967bc96b9a93aeda1218633fd445d743f8c6109 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-a278231, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:42:58.692 INFO:tasks.rados.rados.0.smithi110.stdout:1588: done (3 left) 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1589: rollback oid 20 current snap is 160 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 146 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1585: finishing write tid 2 to smithi11075562-32 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1585: finishing write tid 3 to smithi11075562-32 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1585: finishing write tid 4 to smithi11075562-32 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1585: finishing write tid 5 to smithi11075562-32 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 975 (ObjNum 507 snap 160 seq_num 507) dirty exists 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1585: left oid 32 (ObjNum 507 snap 160 seq_num 507) 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1586: finishing write tid 1 to smithi11075562-36 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1586: finishing write tid 2 to smithi11075562-36 2024-09-14T03:42:58.693 INFO:tasks.rados.rados.0.smithi110.stdout:1586: finishing write tid 3 to smithi11075562-36 2024-09-14T03:42:58.695 INFO:tasks.rados.rados.0.smithi110.stdout:1586: finishing write tid 4 to smithi11075562-36 2024-09-14T03:42:58.695 INFO:tasks.rados.rados.0.smithi110.stdout:1586: finishing write tid 5 to smithi11075562-36 2024-09-14T03:42:58.695 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1124 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-09-14T03:42:58.695 INFO:tasks.rados.rados.0.smithi110.stdout:1586: left oid 36 (ObjNum 508 snap 160 seq_num 508) 2024-09-14T03:42:58.695 INFO:tasks.rados.rados.0.smithi110.stdout:1587: finishing write tid 1 to smithi11075562-22 2024-09-14T03:42:58.695 INFO:tasks.rados.rados.0.smithi110.stdout:1587: finishing write tid 2 to smithi11075562-22 2024-09-14T03:42:58.695 INFO:tasks.rados.rados.0.smithi110.stdout:1587: finishing write tid 3 to smithi11075562-22 2024-09-14T03:42:58.696 INFO:tasks.rados.rados.0.smithi110.stdout:1587: finishing write tid 4 to smithi11075562-22 2024-09-14T03:42:58.696 INFO:tasks.rados.rados.0.smithi110.stdout:1587: finishing write tid 5 to smithi11075562-22 2024-09-14T03:42:58.696 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 980 (ObjNum 509 snap 160 seq_num 509) dirty exists 2024-09-14T03:42:58.696 INFO:tasks.rados.rados.0.smithi110.stdout:1587: left oid 22 (ObjNum 509 snap 160 seq_num 509) 2024-09-14T03:42:58.696 INFO:tasks.rados.rados.0.smithi110.stdout:1585: done (3 left) 2024-09-14T03:42:58.696 INFO:tasks.rados.rados.0.smithi110.stdout:1586: done (2 left) 2024-09-14T03:42:58.696 INFO:tasks.rados.rados.0.smithi110.stdout:1587: done (1 left) 2024-09-14T03:42:58.696 INFO:tasks.rados.rados.0.smithi110.stdout:1590: snap_remove snap 83 2024-09-14T03:42:58.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:58 smithi110 ceph-mon[86423]: osdmap e381: 8 total, 7 up, 8 in 2024-09-14T03:42:58.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:58 smithi110 ceph-mon[86423]: pgmap v198: 121 pgs: 22 active+undersized, 6 peering, 1 stale+active+clean, 6 active+undersized+degraded, 86 active+clean; 459 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 888 KiB/s wr, 5 op/s; 23/783 objects degraded (2.937%) 2024-09-14T03:42:58.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:58 smithi110 ceph-mon[86423]: osdmap e382: 8 total, 7 up, 8 in 2024-09-14T03:42:58.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:58 smithi110 ceph-mon[88262]: osdmap e381: 8 total, 7 up, 8 in 2024-09-14T03:42:58.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:58 smithi110 ceph-mon[88262]: pgmap v198: 121 pgs: 22 active+undersized, 6 peering, 1 stale+active+clean, 6 active+undersized+degraded, 86 active+clean; 459 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 888 KiB/s wr, 5 op/s; 23/783 objects degraded (2.937%) 2024-09-14T03:42:58.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:58 smithi110 ceph-mon[88262]: osdmap e382: 8 total, 7 up, 8 in 2024-09-14T03:42:58.947 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:58 smithi110 podman[102295]: 2024-09-14 03:42:58.667838041 +0000 UTC m=+0.111167464 container create 0cdc58c38711a34029d3f27d974cdbe4f2e83dca3c80164b2e484a24d7e68e05 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-14T03:42:58.947 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:58 smithi110 podman[102295]: 2024-09-14 03:42:58.572248691 +0000 UTC m=+0.015578124 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:42:58.947 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:58 smithi110 podman[102295]: 2024-09-14 03:42:58.887127381 +0000 UTC m=+0.330456805 container init 0cdc58c38711a34029d3f27d974cdbe4f2e83dca3c80164b2e484a24d7e68e05 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-14T03:42:58.947 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:58 smithi110 podman[102295]: 2024-09-14 03:42:58.890440811 +0000 UTC m=+0.333770240 container start 0cdc58c38711a34029d3f27d974cdbe4f2e83dca3c80164b2e484a24d7e68e05 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-14T03:42:58.947 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:58 smithi110 ceph-osd[102314]: -- 172.21.15.110:0/2765132656 <== mon.2 v2:172.21.15.110:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563141ae3a40 con 0x5631428b2400 2024-09-14T03:42:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:58 smithi195 ceph-mon[82388]: osdmap e381: 8 total, 7 up, 8 in 2024-09-14T03:42:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:58 smithi195 ceph-mon[82388]: pgmap v198: 121 pgs: 22 active+undersized, 6 peering, 1 stale+active+clean, 6 active+undersized+degraded, 86 active+clean; 459 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 888 KiB/s wr, 5 op/s; 23/783 objects degraded (2.937%) 2024-09-14T03:42:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:58 smithi195 ceph-mon[82388]: osdmap e382: 8 total, 7 up, 8 in 2024-09-14T03:42:59.446 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:58 smithi110 bash[102295]: 0cdc58c38711a34029d3f27d974cdbe4f2e83dca3c80164b2e484a24d7e68e05 2024-09-14T03:42:59.446 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:42:58 smithi110 systemd[1]: Started Ceph osd.2 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:42:59.694 INFO:tasks.rados.rados.0.smithi110.stdout:1590: done (1 left) 2024-09-14T03:42:59.695 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 160 2024-09-14T03:42:59.695 INFO:tasks.rados.rados.0.smithi110.stdout:1591: seq_num 510 ranges {1286144=548864,1835008=745472,2580480=57344} 2024-09-14T03:42:59.706 INFO:tasks.rados.rados.0.smithi110.stdout:1591: writing smithi11075562-24 from 1286144 to 1835008 tid 1 2024-09-14T03:42:59.711 INFO:tasks.rados.rados.0.smithi110.stdout:1591: writing smithi11075562-24 from 1835008 to 2580480 tid 2 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:1591: writing smithi11075562-24 from 2580480 to 2637824 tid 3 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:1592: read oid 13 snap 90 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:1592: expect (ObjNum 287 snap 86 seq_num 287) 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:1589: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:1589: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:1589: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1005 (ObjNum 445 snap 143 seq_num 445) dirty exists 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:1589: done (2 left) 2024-09-14T03:42:59.712 INFO:tasks.rados.rados.0.smithi110.stdout:1593: delete oid 29 current snap is 160 2024-09-14T03:42:59.713 INFO:tasks.rados.rados.0.smithi110.stdout:1591: finishing write tid 1 to smithi11075562-24 2024-09-14T03:42:59.716 INFO:tasks.rados.rados.0.smithi110.stdout:1593: done (2 left) 2024-09-14T03:42:59.716 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 160 2024-09-14T03:42:59.716 INFO:tasks.rados.rados.0.smithi110.stdout:1594: seq_num 512 ranges {2064384=581632,2646016=516096,3162112=286720} 2024-09-14T03:42:59.730 INFO:tasks.rados.rados.0.smithi110.stdout:1594: writing smithi11075562-33 from 2064384 to 2646016 tid 1 2024-09-14T03:42:59.733 INFO:tasks.rados.rados.0.smithi110.stdout:1594: writing smithi11075562-33 from 2646016 to 3162112 tid 2 2024-09-14T03:42:59.736 INFO:tasks.rados.rados.0.smithi110.stdout:1594: writing smithi11075562-33 from 3162112 to 3448832 tid 3 2024-09-14T03:42:59.736 INFO:tasks.rados.rados.0.smithi110.stdout:1595: rollback oid 26 current snap is 160 2024-09-14T03:42:59.736 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 133 2024-09-14T03:42:59.736 INFO:tasks.rados.rados.0.smithi110.stdout:1591: finishing write tid 2 to smithi11075562-24 2024-09-14T03:42:59.736 INFO:tasks.rados.rados.0.smithi110.stdout:1591: finishing write tid 3 to smithi11075562-24 2024-09-14T03:42:59.736 INFO:tasks.rados.rados.0.smithi110.stdout:1591: finishing write tid 4 to smithi11075562-24 2024-09-14T03:42:59.736 INFO:tasks.rados.rados.0.smithi110.stdout:1591: finishing write tid 5 to smithi11075562-24 2024-09-14T03:42:59.736 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 569 (ObjNum 510 snap 160 seq_num 510) dirty exists 2024-09-14T03:42:59.737 INFO:tasks.rados.rados.0.smithi110.stdout:1591: left oid 24 (ObjNum 510 snap 160 seq_num 510) 2024-09-14T03:42:59.737 INFO:tasks.rados.rados.0.smithi110.stdout:1591: done (3 left) 2024-09-14T03:42:59.737 INFO:tasks.rados.rados.0.smithi110.stdout:1596: copy_from oid 31 from oid 27 current snap is 160 2024-09-14T03:42:59.737 INFO:tasks.rados.rados.0.smithi110.stdout:1597: read oid 42 snap -1 2024-09-14T03:42:59.737 INFO:tasks.rados.rados.0.smithi110.stdout:1597: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:42:59.737 INFO:tasks.rados.rados.0.smithi110.stdout:1598: snap_remove snap 157 2024-09-14T03:42:59.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[88262]: Health check failed: Degraded data redundancy: 23/783 objects degraded (2.937%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:59.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:59.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:59.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:59.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[88262]: osdmap e383: 8 total, 7 up, 8 in 2024-09-14T03:42:59.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[86423]: Health check failed: Degraded data redundancy: 23/783 objects degraded (2.937%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:42:59.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:59.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:42:59.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:42:59.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:42:59 smithi110 ceph-mon[86423]: osdmap e383: 8 total, 7 up, 8 in 2024-09-14T03:43:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:59 smithi195 ceph-mon[82388]: Health check failed: Degraded data redundancy: 23/783 objects degraded (2.937%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:59 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:59 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:59 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:42:59 smithi195 ceph-mon[82388]: osdmap e383: 8 total, 7 up, 8 in 2024-09-14T03:43:00.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:00.720 INFO:tasks.rados.rados.0.smithi110.stdout:1598: done (5 left) 2024-09-14T03:43:00.720 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 160 2024-09-14T03:43:00.720 INFO:tasks.rados.rados.0.smithi110.stdout:1594: finishing write tid 1 to smithi11075562-33 2024-09-14T03:43:00.720 INFO:tasks.rados.rados.0.smithi110.stdout:1599: seq_num 513 ranges {1818624=598016,2416640=794624,3211264=8192} 2024-09-14T03:43:00.733 INFO:tasks.rados.rados.0.smithi110.stdout:1599: writing smithi11075562-35 from 1818624 to 2416640 tid 1 2024-09-14T03:43:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:1599: writing smithi11075562-35 from 2416640 to 3211264 tid 2 2024-09-14T03:43:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:1599: writing smithi11075562-35 from 3211264 to 3219456 tid 3 2024-09-14T03:43:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:1600: read oid 36 snap -1 2024-09-14T03:43:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:1600: expect (ObjNum 508 snap 160 seq_num 508) 2024-09-14T03:43:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 160 2024-09-14T03:43:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:1595: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:43:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:1595: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:43:00.739 INFO:tasks.rados.rados.0.smithi110.stdout:1601: seq_num 514 ranges {6365184=614400,6979584=679936,7659520=122880} 2024-09-14T03:43:00.776 INFO:tasks.rados.rados.0.smithi110.stdout:1601: writing smithi11075562-17 from 6365184 to 6979584 tid 1 2024-09-14T03:43:00.780 INFO:tasks.rados.rados.0.smithi110.stdout:1601: writing smithi11075562-17 from 6979584 to 7659520 tid 2 2024-09-14T03:43:00.781 INFO:tasks.rados.rados.0.smithi110.stdout:1601: writing smithi11075562-17 from 7659520 to 7782400 tid 3 2024-09-14T03:43:00.781 INFO:tasks.rados.rados.0.smithi110.stdout:1602: rollback oid 24 current snap is 160 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 140 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:1595: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 765 (ObjNum 321 snap 104 seq_num 321) dirty exists 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:1594: finishing write tid 2 to smithi11075562-33 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:1594: finishing write tid 3 to smithi11075562-33 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:1594: finishing write tid 4 to smithi11075562-33 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:1594: finishing write tid 5 to smithi11075562-33 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1009 (ObjNum 512 snap 160 seq_num 512) dirty exists 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:1594: left oid 33 (ObjNum 512 snap 160 seq_num 512) 2024-09-14T03:43:00.782 INFO:tasks.rados.rados.0.smithi110.stdout:1592: expect (ObjNum 287 snap 86 seq_num 287) 2024-09-14T03:43:00.881 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:43:00 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:43:00.530+0000 7fba543ee640 -1 Falling back to public interface 2024-09-14T03:43:00.881 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:00 smithi110 ceph-mon[86423]: pgmap v201: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 443 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.5 MiB/s wr, 7 op/s; 74/780 objects degraded (9.487%) 2024-09-14T03:43:00.882 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:00 smithi110 ceph-mon[88262]: pgmap v201: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 443 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.5 MiB/s wr, 7 op/s; 74/780 objects degraded (9.487%) 2024-09-14T03:43:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:00 smithi195 ceph-mon[82388]: pgmap v201: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 443 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.5 MiB/s wr, 7 op/s; 74/780 objects degraded (9.487%) 2024-09-14T03:43:01.274 INFO:tasks.rados.rados.0.smithi110.stdout:1596: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:43:01.274 INFO:tasks.rados.rados.0.smithi110.stdout:1596: finishing copy_from to smithi11075562-31 2024-09-14T03:43:01.274 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 682 (ObjNum 464 snap 150 seq_num 464) dirty exists 2024-09-14T03:43:01.274 INFO:tasks.rados.rados.0.smithi110.stdout:1597: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:43:01.594 INFO:tasks.rados.rados.0.smithi110.stdout:1599: finishing write tid 1 to smithi11075562-35 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1592: done (8 left) 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1594: done (7 left) 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1595: done (6 left) 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1596: done (5 left) 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1597: done (4 left) 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1603: rollback oid 10 current snap is 160 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 156 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1599: finishing write tid 2 to smithi11075562-35 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1599: finishing write tid 3 to smithi11075562-35 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1599: finishing write tid 4 to smithi11075562-35 2024-09-14T03:43:01.595 INFO:tasks.rados.rados.0.smithi110.stdout:1600: expect (ObjNum 508 snap 160 seq_num 508) 2024-09-14T03:43:01.631 INFO:tasks.rados.rados.0.smithi110.stdout:1599: finishing write tid 5 to smithi11075562-35 2024-09-14T03:43:01.631 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1016 (ObjNum 513 snap 160 seq_num 513) dirty exists 2024-09-14T03:43:01.631 INFO:tasks.rados.rados.0.smithi110.stdout:1599: left oid 35 (ObjNum 513 snap 160 seq_num 513) 2024-09-14T03:43:01.631 INFO:tasks.rados.rados.0.smithi110.stdout:1601: finishing write tid 1 to smithi11075562-17 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1599: done (4 left) 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1600: done (3 left) 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1604: rmattr oid 42 current snap is 160 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1602: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1602: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1601: finishing write tid 2 to smithi11075562-17 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1601: finishing write tid 3 to smithi11075562-17 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1602: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 572 (ObjNum 432 snap 138 seq_num 432) dirty exists 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1601: finishing write tid 4 to smithi11075562-17 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1601: finishing write tid 5 to smithi11075562-17 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 899 (ObjNum 514 snap 160 seq_num 514) dirty exists 2024-09-14T03:43:01.632 INFO:tasks.rados.rados.0.smithi110.stdout:1601: left oid 17 (ObjNum 514 snap 160 seq_num 514) 2024-09-14T03:43:01.633 INFO:tasks.rados.rados.0.smithi110.stdout:1601: done (3 left) 2024-09-14T03:43:01.633 INFO:tasks.rados.rados.0.smithi110.stdout:1602: done (2 left) 2024-09-14T03:43:01.633 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 160 2024-09-14T03:43:01.633 INFO:tasks.rados.rados.0.smithi110.stdout:1603: finishing rollback tid 0 to smithi11075562-10 2024-09-14T03:43:01.633 INFO:tasks.rados.rados.0.smithi110.stdout:1603: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:43:01.633 INFO:tasks.rados.rados.0.smithi110.stdout:1603: finishing rollback tid 2 to smithi11075562-10 2024-09-14T03:43:01.633 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 576 (ObjNum 500 snap 156 seq_num 500) dirty exists 2024-09-14T03:43:01.633 INFO:tasks.rados.rados.0.smithi110.stdout:1605: seq_num 515 ranges {352256=630784,983040=565248,1548288=237568} 2024-09-14T03:43:01.636 INFO:tasks.rados.rados.0.smithi110.stdout:1605: writing smithi11075562-20 from 352256 to 983040 tid 1 2024-09-14T03:43:01.639 INFO:tasks.rados.rados.0.smithi110.stdout:1605: writing smithi11075562-20 from 983040 to 1548288 tid 2 2024-09-14T03:43:01.641 INFO:tasks.rados.rados.0.smithi110.stdout:1605: writing smithi11075562-20 from 1548288 to 1785856 tid 3 2024-09-14T03:43:01.641 INFO:tasks.rados.rados.0.smithi110.stdout:1603: done (2 left) 2024-09-14T03:43:01.641 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1017 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:43:01.641 INFO:tasks.rados.rados.0.smithi110.stdout:1605: finishing write tid 1 to smithi11075562-20 2024-09-14T03:43:01.642 INFO:tasks.rados.rados.0.smithi110.stdout:1606: read oid 13 snap -1 2024-09-14T03:43:01.642 INFO:tasks.rados.rados.0.smithi110.stdout:1606: expect (ObjNum 426 snap 136 seq_num 426) 2024-09-14T03:43:01.642 INFO:tasks.rados.rados.0.smithi110.stdout:1604: done (2 left) 2024-09-14T03:43:01.642 INFO:tasks.rados.rados.0.smithi110.stdout:1607: snap_remove snap 156 2024-09-14T03:43:01.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:43:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:43:01] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.51.0" 2024-09-14T03:43:01.722 INFO:tasks.rados.rados.0.smithi110.stdout:1607: done (2 left) 2024-09-14T03:43:01.722 INFO:tasks.rados.rados.0.smithi110.stdout:1608: snap_create 2024-09-14T03:43:01.722 INFO:tasks.rados.rados.0.smithi110.stdout:1605: finishing write tid 2 to smithi11075562-20 2024-09-14T03:43:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:1605: finishing write tid 3 to smithi11075562-20 2024-09-14T03:43:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:1605: finishing write tid 4 to smithi11075562-20 2024-09-14T03:43:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:1605: finishing write tid 5 to smithi11075562-20 2024-09-14T03:43:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1013 (ObjNum 515 snap 160 seq_num 515) dirty exists 2024-09-14T03:43:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:1605: left oid 20 (ObjNum 515 snap 160 seq_num 515) 2024-09-14T03:43:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:1606: expect (ObjNum 426 snap 136 seq_num 426) 2024-09-14T03:43:02.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:01 smithi110 ceph-mon[88262]: osdmap e384: 8 total, 7 up, 8 in 2024-09-14T03:43:02.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:01 smithi110 ceph-mon[86423]: osdmap e384: 8 total, 7 up, 8 in 2024-09-14T03:43:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:01 smithi195 ceph-mon[82388]: osdmap e384: 8 total, 7 up, 8 in 2024-09-14T03:43:02.729 INFO:tasks.rados.rados.0.smithi110.stdout:1605: done (2 left) 2024-09-14T03:43:02.729 INFO:tasks.rados.rados.0.smithi110.stdout:1606: done (1 left) 2024-09-14T03:43:02.729 INFO:tasks.rados.rados.0.smithi110.stdout:1608: done (0 left) 2024-09-14T03:43:02.729 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 161 2024-09-14T03:43:02.730 INFO:tasks.rados.rados.0.smithi110.stdout:1609: seq_num 516 ranges {0=647168,647168=450560,1097728=352256} 2024-09-14T03:43:02.732 INFO:tasks.rados.rados.0.smithi110.stdout:1609: writing smithi11075562-41 from 0 to 647168 tid 1 2024-09-14T03:43:02.735 INFO:tasks.rados.rados.0.smithi110.stdout:1609: writing smithi11075562-41 from 647168 to 1097728 tid 2 2024-09-14T03:43:02.737 INFO:tasks.rados.rados.0.smithi110.stdout:1609: writing smithi11075562-41 from 1097728 to 1449984 tid 3 2024-09-14T03:43:02.738 INFO:tasks.rados.rados.0.smithi110.stdout:1610: snap_remove snap 159 2024-09-14T03:43:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:02 smithi110 ceph-mon[86423]: osdmap e385: 8 total, 7 up, 8 in 2024-09-14T03:43:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:02 smithi110 ceph-mon[86423]: pgmap v204: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 443 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 906 KiB/s wr, 3 op/s; 74/780 objects degraded (9.487%) 2024-09-14T03:43:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:02 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:02 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:02 smithi110 ceph-mon[88262]: osdmap e385: 8 total, 7 up, 8 in 2024-09-14T03:43:03.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:02 smithi110 ceph-mon[88262]: pgmap v204: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 443 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 906 KiB/s wr, 3 op/s; 74/780 objects degraded (9.487%) 2024-09-14T03:43:03.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:02 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:02 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:02 smithi195 ceph-mon[82388]: osdmap e385: 8 total, 7 up, 8 in 2024-09-14T03:43:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:02 smithi195 ceph-mon[82388]: pgmap v204: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 443 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 906 KiB/s wr, 3 op/s; 74/780 objects degraded (9.487%) 2024-09-14T03:43:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:02 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:02 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.732 INFO:tasks.rados.rados.0.smithi110.stdout:1610: done (1 left) 2024-09-14T03:43:03.733 INFO:tasks.rados.rados.0.smithi110.stdout:1611: read oid 7 snap -1 2024-09-14T03:43:03.733 INFO:tasks.rados.rados.0.smithi110.stdout:1611: expect (ObjNum 470 snap 150 seq_num 470) 2024-09-14T03:43:03.733 INFO:tasks.rados.rados.0.smithi110.stdout:1612: read oid 45 snap -1 2024-09-14T03:43:03.733 INFO:tasks.rados.rados.0.smithi110.stdout:1612: expect (ObjNum 457 snap 149 seq_num 457) 2024-09-14T03:43:03.733 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 161 2024-09-14T03:43:03.733 INFO:tasks.rados.rados.0.smithi110.stdout:1609: finishing write tid 1 to smithi11075562-41 2024-09-14T03:43:03.733 INFO:tasks.rados.rados.0.smithi110.stdout:1613: seq_num 517 ranges {1122304=663552,1785856=786432,2572288=16384} 2024-09-14T03:43:03.743 INFO:tasks.rados.rados.0.smithi110.stdout:1613: writing smithi11075562-6 from 1122304 to 1785856 tid 1 2024-09-14T03:43:03.749 INFO:tasks.rados.rados.0.smithi110.stdout:1613: writing smithi11075562-6 from 1785856 to 2572288 tid 2 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1613: writing smithi11075562-6 from 2572288 to 2588672 tid 3 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1614: rollback oid 36 current snap is 161 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 36 to 160 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1609: finishing write tid 2 to smithi11075562-41 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1609: finishing write tid 3 to smithi11075562-41 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1609: finishing write tid 4 to smithi11075562-41 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1609: finishing write tid 5 to smithi11075562-41 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 900 (ObjNum 516 snap 161 seq_num 516) dirty exists 2024-09-14T03:43:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1609: left oid 41 (ObjNum 516 snap 161 seq_num 516) 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1609: done (4 left) 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1615: read oid 18 snap 146 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1615: expect (ObjNum 407 snap 134 seq_num 407) 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1616: read oid 50 snap -1 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1616: expect deleted 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1617: read oid 34 snap -1 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1617: expect deleted 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1618: snap_create 2024-09-14T03:43:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1611: expect (ObjNum 470 snap 150 seq_num 470) 2024-09-14T03:43:03.799 INFO:tasks.rados.rados.0.smithi110.stdout:1613: finishing write tid 1 to smithi11075562-6 2024-09-14T03:43:03.799 INFO:tasks.rados.rados.0.smithi110.stdout:1613: finishing write tid 2 to smithi11075562-6 2024-09-14T03:43:03.799 INFO:tasks.rados.rados.0.smithi110.stdout:1613: finishing write tid 3 to smithi11075562-6 2024-09-14T03:43:03.799 INFO:tasks.rados.rados.0.smithi110.stdout:1613: finishing write tid 4 to smithi11075562-6 2024-09-14T03:43:03.799 INFO:tasks.rados.rados.0.smithi110.stdout:1612: expect (ObjNum 457 snap 149 seq_num 457) 2024-09-14T03:43:03.901 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:03 smithi110 ceph-mon[88262]: osdmap e386: 8 total, 7 up, 8 in 2024-09-14T03:43:03.901 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:03 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.901 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:03 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.901 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:03 smithi110 ceph-mon[86423]: osdmap e386: 8 total, 7 up, 8 in 2024-09-14T03:43:03.901 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:03 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:03.901 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:03 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:04.052 INFO:tasks.rados.rados.0.smithi110.stdout:1614: finishing rollback tid 0 to smithi11075562-36 2024-09-14T03:43:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:1614: finishing rollback tid 1 to smithi11075562-36 2024-09-14T03:43:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:1614: finishing rollback tid 2 to smithi11075562-36 2024-09-14T03:43:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1142 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-09-14T03:43:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:1613: finishing write tid 5 to smithi11075562-6 2024-09-14T03:43:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 899 (ObjNum 517 snap 161 seq_num 517) dirty exists 2024-09-14T03:43:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:1613: left oid 6 (ObjNum 517 snap 161 seq_num 517) 2024-09-14T03:43:04.053 INFO:tasks.rados.rados.0.smithi110.stdout:1615: expect (ObjNum 407 snap 134 seq_num 407) 2024-09-14T03:43:04.148 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:03 smithi195 ceph-mon[82388]: osdmap e386: 8 total, 7 up, 8 in 2024-09-14T03:43:04.148 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:03 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:04.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:03 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:04.446 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:43:04 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:43:04.190+0000 7fba543ee640 -1 osd.2 376 log_to_monitors true 2024-09-14T03:43:04.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:04.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:43:04.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:04.739 INFO:tasks.rados.rados.0.smithi110.stdout:1611: done (7 left) 2024-09-14T03:43:04.739 INFO:tasks.rados.rados.0.smithi110.stdout:1612: done (6 left) 2024-09-14T03:43:04.739 INFO:tasks.rados.rados.0.smithi110.stdout:1613: done (5 left) 2024-09-14T03:43:04.739 INFO:tasks.rados.rados.0.smithi110.stdout:1614: done (4 left) 2024-09-14T03:43:04.739 INFO:tasks.rados.rados.0.smithi110.stdout:1615: done (3 left) 2024-09-14T03:43:04.739 INFO:tasks.rados.rados.0.smithi110.stdout:1616: done (2 left) 2024-09-14T03:43:04.740 INFO:tasks.rados.rados.0.smithi110.stdout:1617: done (1 left) 2024-09-14T03:43:04.740 INFO:tasks.rados.rados.0.smithi110.stdout:1618: done (0 left) 2024-09-14T03:43:04.740 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 162 2024-09-14T03:43:04.740 INFO:tasks.rados.rados.0.smithi110.stdout:1619: seq_num 518 ranges {7512064=679936,8192000=671744,8863744=131072} 2024-09-14T03:43:04.783 INFO:tasks.rados.rados.0.smithi110.stdout:1619: writing smithi11075562-25 from 7512064 to 8192000 tid 1 2024-09-14T03:43:04.787 INFO:tasks.rados.rados.0.smithi110.stdout:1619: writing smithi11075562-25 from 8192000 to 8863744 tid 2 2024-09-14T03:43:04.788 INFO:tasks.rados.rados.0.smithi110.stdout:1619: writing smithi11075562-25 from 8863744 to 8994816 tid 3 2024-09-14T03:43:04.788 INFO:tasks.rados.rados.0.smithi110.stdout:1620: copy_from oid 31 from oid 13 current snap is 162 2024-09-14T03:43:04.788 INFO:tasks.rados.rados.0.smithi110.stdout:1621: snap_remove snap 142 2024-09-14T03:43:04.923 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:43:04 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:43:04.760+0000 7fba4c198640 -1 osd.2 376 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:43:04.923 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:04 smithi110 ceph-mon[88262]: osdmap e387: 8 total, 7 up, 8 in 2024-09-14T03:43:04.923 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:04 smithi110 ceph-mon[88262]: pgmap v207: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 455 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 14 op/s; 76/765 objects degraded (9.935%) 2024-09-14T03:43:04.923 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:04 smithi110 ceph-mon[88262]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T03:43:04.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:04 smithi110 ceph-mon[86423]: osdmap e387: 8 total, 7 up, 8 in 2024-09-14T03:43:04.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:04 smithi110 ceph-mon[86423]: pgmap v207: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 455 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 14 op/s; 76/765 objects degraded (9.935%) 2024-09-14T03:43:04.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:04 smithi110 ceph-mon[86423]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T03:43:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:04 smithi195 ceph-mon[82388]: osdmap e387: 8 total, 7 up, 8 in 2024-09-14T03:43:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:04 smithi195 ceph-mon[82388]: pgmap v207: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 455 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 14 op/s; 76/765 objects degraded (9.935%) 2024-09-14T03:43:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:04 smithi195 ceph-mon[82388]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T03:43:05.742 INFO:tasks.rados.rados.0.smithi110.stdout:1621: done (2 left) 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1622: read oid 28 snap -1 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1622: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1619: finishing write tid 1 to smithi11075562-25 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1619: finishing write tid 2 to smithi11075562-25 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1619: finishing write tid 3 to smithi11075562-25 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1619: finishing write tid 4 to smithi11075562-25 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1619: finishing write tid 5 to smithi11075562-25 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1147 (ObjNum 518 snap 162 seq_num 518) dirty exists 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1619: left oid 25 (ObjNum 518 snap 162 seq_num 518) 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1620: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:1620: finishing copy_from to smithi11075562-31 2024-09-14T03:43:05.743 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 899 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:43:05.744 INFO:tasks.rados.rados.0.smithi110.stdout:1619: done (2 left) 2024-09-14T03:43:05.744 INFO:tasks.rados.rados.0.smithi110.stdout:1620: done (1 left) 2024-09-14T03:43:05.744 INFO:tasks.rados.rados.0.smithi110.stdout:1623: copy_from oid 24 from oid 31 current snap is 162 2024-09-14T03:43:05.744 INFO:tasks.rados.rados.0.smithi110.stdout:1624: snap_create 2024-09-14T03:43:05.755 INFO:tasks.rados.rados.0.smithi110.stdout:1622: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:43:06.080 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:05 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 76/765 objects degraded (9.935%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:06.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:05 smithi110 ceph-mon[88262]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:43:06.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:05 smithi110 ceph-mon[88262]: osdmap e388: 8 total, 7 up, 8 in 2024-09-14T03:43:06.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:05 smithi110 ceph-mon[88262]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:43:06.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:05 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 76/765 objects degraded (9.935%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:06.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:05 smithi110 ceph-mon[86423]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:43:06.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:05 smithi110 ceph-mon[86423]: osdmap e388: 8 total, 7 up, 8 in 2024-09-14T03:43:06.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:05 smithi110 ceph-mon[86423]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:43:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:05 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 76/765 objects degraded (9.935%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:05 smithi195 ceph-mon[82388]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:43:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:05 smithi195 ceph-mon[82388]: osdmap e388: 8 total, 7 up, 8 in 2024-09-14T03:43:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:05 smithi195 ceph-mon[82388]: from='osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:43:06.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:06.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:06.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:43:06.517+0000 7f940e88b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-14T03:43:06.743 INFO:tasks.rados.rados.0.smithi110.stdout:1622: done (2 left) 2024-09-14T03:43:06.743 INFO:tasks.rados.rados.0.smithi110.stdout:1624: done (1 left) 2024-09-14T03:43:06.743 INFO:tasks.rados.rados.0.smithi110.stdout:1625: snap_remove snap 135 2024-09-14T03:43:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:43:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625] boot 2024-09-14T03:43:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: pgmap v210: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 460 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.1 MiB/s wr, 16 op/s; 77/762 objects degraded (10.105%) 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[88262]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625] boot 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: pgmap v210: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 460 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.1 MiB/s wr, 16 op/s; 77/762 objects degraded (10.105%) 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:43:07.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:07.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:43:07.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:06 smithi110 ceph-mon[86423]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:43:07.208 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: osd.2 [v2:172.21.15.110:6818/2442181625,v1:172.21.15.110:6819/2442181625] boot 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: pgmap v210: 121 pgs: 28 active+undersized, 9 active+undersized+degraded, 84 active+clean; 460 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.1 MiB/s wr, 16 op/s; 77/762 objects degraded (10.105%) 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:43:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:07.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:07.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:07.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:43:07.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:06 smithi195 ceph-mon[82388]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:43:07.745 INFO:tasks.rados.rados.0.smithi110.stdout:1625: done (1 left) 2024-09-14T03:43:07.745 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 163 2024-09-14T03:43:07.745 INFO:tasks.rados.rados.0.smithi110.stdout:1626: seq_num 519 ranges {2973696=696320,3670016=548864,4218880=253952} 2024-09-14T03:43:07.766 INFO:tasks.rados.rados.0.smithi110.stdout:1626: writing smithi11075562-38 from 2973696 to 3670016 tid 1 2024-09-14T03:43:07.770 INFO:tasks.rados.rados.0.smithi110.stdout:1626: writing smithi11075562-38 from 3670016 to 4218880 tid 2 2024-09-14T03:43:07.773 INFO:tasks.rados.rados.0.smithi110.stdout:1626: writing smithi11075562-38 from 4218880 to 4472832 tid 3 2024-09-14T03:43:07.773 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 163 2024-09-14T03:43:07.773 INFO:tasks.rados.rados.0.smithi110.stdout:1623: finishing copy_from racing read to smithi11075562-24 2024-09-14T03:43:07.773 INFO:tasks.rados.rados.0.smithi110.stdout:1623: finishing copy_from to smithi11075562-24 2024-09-14T03:43:07.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 899 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:43:07.773 INFO:tasks.rados.rados.0.smithi110.stdout:1626: finishing write tid 1 to smithi11075562-38 2024-09-14T03:43:07.774 INFO:tasks.rados.rados.0.smithi110.stdout:1627: seq_num 520 ranges {581632=712704,1294336=434176,1728512=368640} 2024-09-14T03:43:07.779 INFO:tasks.rados.rados.0.smithi110.stdout:1627: writing smithi11075562-26 from 581632 to 1294336 tid 1 2024-09-14T03:43:07.782 INFO:tasks.rados.rados.0.smithi110.stdout:1627: writing smithi11075562-26 from 1294336 to 1728512 tid 2 2024-09-14T03:43:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:1627: writing smithi11075562-26 from 1728512 to 2097152 tid 3 2024-09-14T03:43:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:1623: done (2 left) 2024-09-14T03:43:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:1628: rmattr oid 20 current snap is 163 2024-09-14T03:43:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:1626: finishing write tid 2 to smithi11075562-38 2024-09-14T03:43:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 163 2024-09-14T03:43:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:1626: finishing write tid 3 to smithi11075562-38 2024-09-14T03:43:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:1626: finishing write tid 4 to smithi11075562-38 2024-09-14T03:43:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:1626: finishing write tid 5 to smithi11075562-38 2024-09-14T03:43:07.786 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 527 (ObjNum 519 snap 163 seq_num 519) dirty exists 2024-09-14T03:43:07.786 INFO:tasks.rados.rados.0.smithi110.stdout:1626: left oid 38 (ObjNum 519 snap 163 seq_num 519) 2024-09-14T03:43:07.786 INFO:tasks.rados.rados.0.smithi110.stdout:1627: finishing write tid 1 to smithi11075562-26 2024-09-14T03:43:07.786 INFO:tasks.rados.rados.0.smithi110.stdout:1629: seq_num 521 ranges {2588672=729088,3317760=720896,4038656=81920} 2024-09-14T03:43:07.802 INFO:tasks.rados.rados.0.smithi110.stdout:1629: writing smithi11075562-6 from 2588672 to 3317760 tid 1 2024-09-14T03:43:07.807 INFO:tasks.rados.rados.0.smithi110.stdout:1629: writing smithi11075562-6 from 3317760 to 4038656 tid 2 2024-09-14T03:43:07.807 INFO:tasks.rados.rados.0.smithi110.stdout:1629: writing smithi11075562-6 from 4038656 to 4120576 tid 3 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:1626: done (3 left) 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:1630: delete oid 23 current snap is 163 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:1627: finishing write tid 2 to smithi11075562-26 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:1627: finishing write tid 3 to smithi11075562-26 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1014 (ObjNum 515 snap 160 seq_num 515) dirty exists 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:1627: finishing write tid 4 to smithi11075562-26 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:1627: finishing write tid 5 to smithi11075562-26 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 769 (ObjNum 520 snap 163 seq_num 520) dirty exists 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:1627: left oid 26 (ObjNum 520 snap 163 seq_num 520) 2024-09-14T03:43:07.808 INFO:tasks.rados.rados.0.smithi110.stdout:1629: finishing write tid 1 to smithi11075562-6 2024-09-14T03:43:07.809 INFO:tasks.rados.rados.0.smithi110.stdout:1627: done (3 left) 2024-09-14T03:43:07.810 INFO:tasks.rados.rados.0.smithi110.stdout:1628: done (2 left) 2024-09-14T03:43:07.810 INFO:tasks.rados.rados.0.smithi110.stdout:1630: done (1 left) 2024-09-14T03:43:07.810 INFO:tasks.rados.rados.0.smithi110.stdout:1631: snap_remove snap 161 2024-09-14T03:43:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:07 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:43:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:07 smithi110 ceph-mon[86423]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-14T03:43:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:07 smithi110 ceph-mon[86423]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:43:08.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:07 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:43:08.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:07 smithi110 ceph-mon[88262]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-14T03:43:08.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:07 smithi110 ceph-mon[88262]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:43:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:07 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:43:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:07 smithi195 ceph-mon[82388]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-14T03:43:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:07 smithi195 ceph-mon[82388]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:43:08.771 INFO:tasks.rados.rados.0.smithi110.stdout:1631: done (1 left) 2024-09-14T03:43:08.772 INFO:tasks.rados.rados.0.smithi110.stdout:1632: setattr oid 23 current snap is 163 2024-09-14T03:43:08.772 INFO:tasks.rados.rados.0.smithi110.stdout:1629: finishing write tid 2 to smithi11075562-6 2024-09-14T03:43:08.772 INFO:tasks.rados.rados.0.smithi110.stdout:1629: finishing write tid 3 to smithi11075562-6 2024-09-14T03:43:08.772 INFO:tasks.rados.rados.0.smithi110.stdout:1629: finishing write tid 4 to smithi11075562-6 2024-09-14T03:43:08.772 INFO:tasks.rados.rados.0.smithi110.stdout:1629: finishing write tid 5 to smithi11075562-6 2024-09-14T03:43:08.772 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 920 (ObjNum 521 snap 163 seq_num 521) dirty exists 2024-09-14T03:43:08.772 INFO:tasks.rados.rados.0.smithi110.stdout:1629: left oid 6 (ObjNum 521 snap 163 seq_num 521) 2024-09-14T03:43:08.773 INFO:tasks.rados.rados.0.smithi110.stdout:1629: done (1 left) 2024-09-14T03:43:08.773 INFO:tasks.rados.rados.0.smithi110.stdout:1633: copy_from oid 42 from oid 28 current snap is 163 2024-09-14T03:43:08.773 INFO:tasks.rados.rados.0.smithi110.stdout:1634: read oid 26 snap -1 2024-09-14T03:43:08.773 INFO:tasks.rados.rados.0.smithi110.stdout:1634: expect (ObjNum 520 snap 163 seq_num 520) 2024-09-14T03:43:08.773 INFO:tasks.rados.rados.0.smithi110.stdout:1635: read oid 35 snap -1 2024-09-14T03:43:08.773 INFO:tasks.rados.rados.0.smithi110.stdout:1635: expect (ObjNum 513 snap 160 seq_num 513) 2024-09-14T03:43:08.773 INFO:tasks.rados.rados.0.smithi110.stdout:1636: rollback oid 3 current snap is 163 2024-09-14T03:43:08.773 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 3 to 160 2024-09-14T03:43:08.774 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 163 2024-09-14T03:43:08.774 INFO:tasks.rados.rados.0.smithi110.stdout:1637: seq_num 523 ranges {8036352=770048,8806400=491520,9297920=311296} 2024-09-14T03:43:08.821 INFO:tasks.rados.rados.0.smithi110.stdout:1637: writing smithi11075562-1 from 8036352 to 8806400 tid 1 2024-09-14T03:43:08.824 INFO:tasks.rados.rados.0.smithi110.stdout:1637: writing smithi11075562-1 from 8806400 to 9297920 tid 2 2024-09-14T03:43:08.826 INFO:tasks.rados.rados.0.smithi110.stdout:1637: writing smithi11075562-1 from 9297920 to 9609216 tid 3 2024-09-14T03:43:08.826 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 163 2024-09-14T03:43:08.826 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 901 (ObjNum 129 snap 0 seq_num 66710768) dirty exists 2024-09-14T03:43:08.826 INFO:tasks.rados.rados.0.smithi110.stdout:1638: seq_num 524 ranges {1728512=786432,2514944=778240,3293184=24576} 2024-09-14T03:43:08.839 INFO:tasks.rados.rados.0.smithi110.stdout:1638: writing smithi11075562-11 from 1728512 to 2514944 tid 1 2024-09-14T03:43:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:1638: writing smithi11075562-11 from 2514944 to 3293184 tid 2 2024-09-14T03:43:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:1638: writing smithi11075562-11 from 3293184 to 3317760 tid 3 2024-09-14T03:43:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:1632: done (6 left) 2024-09-14T03:43:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:1639: copy_from oid 14 from oid 10 current snap is 163 2024-09-14T03:43:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:1636: finishing rollback tid 1 to smithi11075562-3 2024-09-14T03:43:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 679 (ObjNum 129 snap 0 seq_num 65182048) dirty dne 2024-09-14T03:43:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:1633: finishing copy_from racing read to smithi11075562-42 2024-09-14T03:43:08.845 INFO:tasks.rados.rados.0.smithi110.stdout:1634: expect (ObjNum 520 snap 163 seq_num 520) 2024-09-14T03:43:08.913 INFO:tasks.rados.rados.0.smithi110.stdout:1633: finishing copy_from to smithi11075562-42 2024-09-14T03:43:08.913 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 508 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:43:08.913 INFO:tasks.rados.rados.0.smithi110.stdout:1635: expect (ObjNum 513 snap 160 seq_num 513) 2024-09-14T03:43:09.048 INFO:tasks.rados.rados.0.smithi110.stdout:1637: finishing write tid 1 to smithi11075562-1 2024-09-14T03:43:09.048 INFO:tasks.rados.rados.0.smithi110.stdout:1637: finishing write tid 2 to smithi11075562-1 2024-09-14T03:43:09.048 INFO:tasks.rados.rados.0.smithi110.stdout:1637: finishing write tid 3 to smithi11075562-1 2024-09-14T03:43:09.048 INFO:tasks.rados.rados.0.smithi110.stdout:1637: finishing write tid 4 to smithi11075562-1 2024-09-14T03:43:09.048 INFO:tasks.rados.rados.0.smithi110.stdout:1637: finishing write tid 5 to smithi11075562-1 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1021 (ObjNum 523 snap 163 seq_num 523) dirty exists 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1637: left oid 1 (ObjNum 523 snap 163 seq_num 523) 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1638: finishing write tid 1 to smithi11075562-11 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1638: finishing write tid 2 to smithi11075562-11 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1633: done (6 left) 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1634: done (5 left) 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1635: done (4 left) 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1636: done (3 left) 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1637: done (2 left) 2024-09-14T03:43:09.049 INFO:tasks.rados.rados.0.smithi110.stdout:1640: snap_remove snap 155 2024-09-14T03:43:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:08 smithi110 ceph-mon[86423]: pgmap v213: 121 pgs: 11 active+undersized, 6 active+undersized+degraded, 104 active+clean; 464 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.4 MiB/s wr, 9 op/s; 71/774 objects degraded (9.173%) 2024-09-14T03:43:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:08 smithi110 ceph-mon[88262]: pgmap v213: 121 pgs: 11 active+undersized, 6 active+undersized+degraded, 104 active+clean; 464 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.4 MiB/s wr, 9 op/s; 71/774 objects degraded (9.173%) 2024-09-14T03:43:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:08 smithi195 ceph-mon[82388]: pgmap v213: 121 pgs: 11 active+undersized, 6 active+undersized+degraded, 104 active+clean; 464 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.4 MiB/s wr, 9 op/s; 71/774 objects degraded (9.173%) 2024-09-14T03:43:09.777 INFO:tasks.rados.rados.0.smithi110.stdout:1640: done (2 left) 2024-09-14T03:43:09.777 INFO:tasks.rados.rados.0.smithi110.stdout:1641: snap_create 2024-09-14T03:43:09.777 INFO:tasks.rados.rados.0.smithi110.stdout:1638: finishing write tid 3 to smithi11075562-11 2024-09-14T03:43:09.777 INFO:tasks.rados.rados.0.smithi110.stdout:1638: finishing write tid 4 to smithi11075562-11 2024-09-14T03:43:09.777 INFO:tasks.rados.rados.0.smithi110.stdout:1638: finishing write tid 5 to smithi11075562-11 2024-09-14T03:43:09.777 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 903 (ObjNum 524 snap 163 seq_num 524) dirty exists 2024-09-14T03:43:09.777 INFO:tasks.rados.rados.0.smithi110.stdout:1638: left oid 11 (ObjNum 524 snap 163 seq_num 524) 2024-09-14T03:43:09.778 INFO:tasks.rados.rados.0.smithi110.stdout:1639: finishing copy_from racing read to smithi11075562-14 2024-09-14T03:43:09.778 INFO:tasks.rados.rados.0.smithi110.stdout:1639: finishing copy_from to smithi11075562-14 2024-09-14T03:43:09.778 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 576 (ObjNum 500 snap 156 seq_num 500) dirty exists 2024-09-14T03:43:10.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:09 smithi110 ceph-mon[86423]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T03:43:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:09 smithi110 ceph-mon[88262]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T03:43:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:09 smithi195 ceph-mon[82388]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T03:43:10.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:10.781 INFO:tasks.rados.rados.0.smithi110.stdout:1638: done (2 left) 2024-09-14T03:43:10.782 INFO:tasks.rados.rados.0.smithi110.stdout:1639: done (1 left) 2024-09-14T03:43:10.782 INFO:tasks.rados.rados.0.smithi110.stdout:1641: done (0 left) 2024-09-14T03:43:10.782 INFO:tasks.rados.rados.0.smithi110.stdout:1642: delete oid 39 current snap is 164 2024-09-14T03:43:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:1642: done (0 left) 2024-09-14T03:43:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:1643: snap_remove snap 158 2024-09-14T03:43:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:10 smithi110 ceph-mon[88262]: pgmap v215: 121 pgs: 8 active+undersized, 6 active+undersized+degraded, 107 active+clean; 464 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.0 MiB/s wr, 8 op/s; 71/768 objects degraded (9.245%) 2024-09-14T03:43:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:10 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 71/768 objects degraded (9.245%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:10 smithi110 ceph-mon[88262]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T03:43:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:10 smithi110 ceph-mon[86423]: pgmap v215: 121 pgs: 8 active+undersized, 6 active+undersized+degraded, 107 active+clean; 464 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.0 MiB/s wr, 8 op/s; 71/768 objects degraded (9.245%) 2024-09-14T03:43:11.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:10 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 71/768 objects degraded (9.245%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:11.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:10 smithi110 ceph-mon[86423]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T03:43:11.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:10 smithi195 ceph-mon[82388]: pgmap v215: 121 pgs: 8 active+undersized, 6 active+undersized+degraded, 107 active+clean; 464 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.0 MiB/s wr, 8 op/s; 71/768 objects degraded (9.245%) 2024-09-14T03:43:11.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:10 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 71/768 objects degraded (9.245%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:11.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:10 smithi195 ceph-mon[82388]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T03:43:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:1643: done (0 left) 2024-09-14T03:43:11.790 INFO:tasks.rados.rados.0.smithi110.stdout:1644: snap_remove snap 162 2024-09-14T03:43:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:11 smithi110 ceph-mon[86423]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T03:43:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:11 smithi110 ceph-mon[86423]: pgmap v218: 121 pgs: 121 active+clean; 479 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.0 MiB/s wr, 9 op/s; 17 MiB/s, 2 objects/s recovering 2024-09-14T03:43:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:11 smithi110 ceph-mon[88262]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T03:43:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:11 smithi110 ceph-mon[88262]: pgmap v218: 121 pgs: 121 active+clean; 479 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.0 MiB/s wr, 9 op/s; 17 MiB/s, 2 objects/s recovering 2024-09-14T03:43:11.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:43:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:43:11] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.51.0" 2024-09-14T03:43:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:11 smithi195 ceph-mon[82388]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T03:43:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:11 smithi195 ceph-mon[82388]: pgmap v218: 121 pgs: 121 active+clean; 479 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.0 MiB/s wr, 9 op/s; 17 MiB/s, 2 objects/s recovering 2024-09-14T03:43:12.788 INFO:tasks.rados.rados.0.smithi110.stdout:1644: done (0 left) 2024-09-14T03:43:12.788 INFO:tasks.rados.rados.0.smithi110.stdout:1645: rollback oid 19 current snap is 164 2024-09-14T03:43:12.789 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 145 2024-09-14T03:43:12.789 INFO:tasks.rados.rados.0.smithi110.stdout:1646: rmattr oid 33 current snap is 164 2024-09-14T03:43:12.789 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 164 2024-09-14T03:43:12.789 INFO:tasks.rados.rados.0.smithi110.stdout:1647: seq_num 526 ranges {7413760=417792,7831552=589824,8421376=614400} 2024-09-14T03:43:12.830 INFO:tasks.rados.rados.0.smithi110.stdout:1647: writing smithi11075562-46 from 7413760 to 7831552 tid 1 2024-09-14T03:43:12.833 INFO:tasks.rados.rados.0.smithi110.stdout:1647: writing smithi11075562-46 from 7831552 to 8421376 tid 2 2024-09-14T03:43:12.837 INFO:tasks.rados.rados.0.smithi110.stdout:1647: writing smithi11075562-46 from 8421376 to 9035776 tid 3 2024-09-14T03:43:12.837 INFO:tasks.rados.rados.0.smithi110.stdout:1648: rmattr oid 40 current snap is 164 2024-09-14T03:43:12.837 INFO:tasks.rados.rados.0.smithi110.stdout:1648: done (3 left) 2024-09-14T03:43:12.838 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 164 2024-09-14T03:43:12.838 INFO:tasks.rados.rados.0.smithi110.stdout:1645: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:43:12.838 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 714 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:43:12.838 INFO:tasks.rados.rados.0.smithi110.stdout:1649: seq_num 527 ranges {0=434176,434176=475136,909312=565248,1474560=163840} 2024-09-14T03:43:12.839 INFO:tasks.rados.rados.0.smithi110.stdout:1649: writing smithi11075562-3 from 0 to 434176 tid 1 2024-09-14T03:43:12.842 INFO:tasks.rados.rados.0.smithi110.stdout:1649: writing smithi11075562-3 from 434176 to 909312 tid 2 2024-09-14T03:43:12.845 INFO:tasks.rados.rados.0.smithi110.stdout:1649: writing smithi11075562-3 from 909312 to 1474560 tid 3 2024-09-14T03:43:12.846 INFO:tasks.rados.rados.0.smithi110.stdout:1649: writing smithi11075562-3 from 1474560 to 1638400 tid 4 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1645: done (3 left) 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 164 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1015 (ObjNum 512 snap 160 seq_num 512) dirty exists 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1647: finishing write tid 1 to smithi11075562-46 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1647: finishing write tid 2 to smithi11075562-46 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1647: finishing write tid 3 to smithi11075562-46 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1647: finishing write tid 4 to smithi11075562-46 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1647: finishing write tid 5 to smithi11075562-46 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1019 (ObjNum 526 snap 164 seq_num 526) dirty exists 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1647: left oid 46 (ObjNum 526 snap 164 seq_num 526) 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1649: finishing write tid 1 to smithi11075562-3 2024-09-14T03:43:12.847 INFO:tasks.rados.rados.0.smithi110.stdout:1649: finishing write tid 2 to smithi11075562-3 2024-09-14T03:43:12.848 INFO:tasks.rados.rados.0.smithi110.stdout:1650: seq_num 528 ranges {0=450560,450560=761856,1212416=442368} 2024-09-14T03:43:12.849 INFO:tasks.rados.rados.0.smithi110.stdout:1650: writing smithi11075562-18 from 0 to 450560 tid 1 2024-09-14T03:43:12.853 INFO:tasks.rados.rados.0.smithi110.stdout:1650: writing smithi11075562-18 from 450560 to 1212416 tid 2 2024-09-14T03:43:12.856 INFO:tasks.rados.rados.0.smithi110.stdout:1650: writing smithi11075562-18 from 1212416 to 1654784 tid 3 2024-09-14T03:43:12.856 INFO:tasks.rados.rados.0.smithi110.stdout:1646: done (3 left) 2024-09-14T03:43:12.856 INFO:tasks.rados.rados.0.smithi110.stdout:1647: done (2 left) 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1651: read oid 15 snap -1 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1651: expect (ObjNum 489 snap 154 seq_num 489) 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 164 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1649: finishing write tid 3 to smithi11075562-3 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1649: finishing write tid 4 to smithi11075562-3 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1649: finishing write tid 5 to smithi11075562-3 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1649: finishing write tid 6 to smithi11075562-3 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 691 (ObjNum 527 snap 164 seq_num 527) dirty exists 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1649: left oid 3 (ObjNum 527 snap 164 seq_num 527) 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1650: finishing write tid 1 to smithi11075562-18 2024-09-14T03:43:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:1652: seq_num 529 ranges {0=466944,466944=647168,1114112=532480,1646592=24576} 2024-09-14T03:43:12.859 INFO:tasks.rados.rados.0.smithi110.stdout:1652: writing smithi11075562-47 from 0 to 466944 tid 1 2024-09-14T03:43:12.863 INFO:tasks.rados.rados.0.smithi110.stdout:1652: writing smithi11075562-47 from 466944 to 1114112 tid 2 2024-09-14T03:43:12.867 INFO:tasks.rados.rados.0.smithi110.stdout:1652: writing smithi11075562-47 from 1114112 to 1646592 tid 3 2024-09-14T03:43:12.867 INFO:tasks.rados.rados.0.smithi110.stdout:1652: writing smithi11075562-47 from 1646592 to 1671168 tid 4 2024-09-14T03:43:12.867 INFO:tasks.rados.rados.0.smithi110.stdout:1649: done (3 left) 2024-09-14T03:43:12.867 INFO:tasks.rados.rados.0.smithi110.stdout:1653: snap_remove snap 163 2024-09-14T03:43:12.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[86423]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/768 objects degraded (9.245%), 6 pgs degraded) 2024-09-14T03:43:12.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:43:12.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[86423]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T03:43:12.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:12.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:12.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[88262]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/768 objects degraded (9.245%), 6 pgs degraded) 2024-09-14T03:43:12.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:43:12.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[88262]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T03:43:12.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:12.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:12 smithi195 ceph-mon[82388]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/768 objects degraded (9.245%), 6 pgs degraded) 2024-09-14T03:43:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:12 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:43:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:12 smithi195 ceph-mon[82388]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T03:43:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:12 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:13.696 INFO:tasks.rados.rados.0.smithi110.stdout:1653: done (3 left) 2024-09-14T03:43:13.696 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 164 2024-09-14T03:43:13.696 INFO:tasks.rados.rados.0.smithi110.stdout:1654: seq_num 530 ranges {4218880=483328,4702208=532480,5234688=671744} 2024-09-14T03:43:13.720 INFO:tasks.rados.rados.0.smithi110.stdout:1654: writing smithi11075562-40 from 4218880 to 4702208 tid 1 2024-09-14T03:43:13.723 INFO:tasks.rados.rados.0.smithi110.stdout:1654: writing smithi11075562-40 from 4702208 to 5234688 tid 2 2024-09-14T03:43:13.728 INFO:tasks.rados.rados.0.smithi110.stdout:1654: writing smithi11075562-40 from 5234688 to 5906432 tid 3 2024-09-14T03:43:13.728 INFO:tasks.rados.rados.0.smithi110.stdout:1655: snap_create 2024-09-14T03:43:13.728 INFO:tasks.rados.rados.0.smithi110.stdout:1650: finishing write tid 2 to smithi11075562-18 2024-09-14T03:43:13.728 INFO:tasks.rados.rados.0.smithi110.stdout:1650: finishing write tid 3 to smithi11075562-18 2024-09-14T03:43:13.728 INFO:tasks.rados.rados.0.smithi110.stdout:1650: finishing write tid 4 to smithi11075562-18 2024-09-14T03:43:13.729 INFO:tasks.rados.rados.0.smithi110.stdout:1650: finishing write tid 5 to smithi11075562-18 2024-09-14T03:43:13.729 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 935 (ObjNum 528 snap 164 seq_num 528) dirty exists 2024-09-14T03:43:13.729 INFO:tasks.rados.rados.0.smithi110.stdout:1650: left oid 18 (ObjNum 528 snap 164 seq_num 528) 2024-09-14T03:43:13.729 INFO:tasks.rados.rados.0.smithi110.stdout:1651: expect (ObjNum 489 snap 154 seq_num 489) 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1652: finishing write tid 1 to smithi11075562-47 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1652: finishing write tid 2 to smithi11075562-47 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1652: finishing write tid 3 to smithi11075562-47 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1652: finishing write tid 4 to smithi11075562-47 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1652: finishing write tid 5 to smithi11075562-47 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1652: finishing write tid 6 to smithi11075562-47 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1174 (ObjNum 529 snap 164 seq_num 529) dirty exists 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1652: left oid 47 (ObjNum 529 snap 164 seq_num 529) 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1654: finishing write tid 1 to smithi11075562-40 2024-09-14T03:43:13.757 INFO:tasks.rados.rados.0.smithi110.stdout:1654: finishing write tid 2 to smithi11075562-40 2024-09-14T03:43:13.758 INFO:tasks.rados.rados.0.smithi110.stdout:1654: finishing write tid 3 to smithi11075562-40 2024-09-14T03:43:13.758 INFO:tasks.rados.rados.0.smithi110.stdout:1654: finishing write tid 4 to smithi11075562-40 2024-09-14T03:43:13.758 INFO:tasks.rados.rados.0.smithi110.stdout:1654: finishing write tid 5 to smithi11075562-40 2024-09-14T03:43:13.758 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 940 (ObjNum 530 snap 164 seq_num 530) dirty exists 2024-09-14T03:43:13.758 INFO:tasks.rados.rados.0.smithi110.stdout:1654: left oid 40 (ObjNum 530 snap 164 seq_num 530) 2024-09-14T03:43:14.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:13 smithi195 ceph-mon[82388]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T03:43:14.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:13 smithi195 ceph-mon[82388]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T03:43:14.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:13 smithi195 ceph-mon[82388]: pgmap v222: 121 pgs: 121 active+clean; 491 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.9 MiB/s wr, 10 op/s 2024-09-14T03:43:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:13 smithi110 ceph-mon[86423]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T03:43:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:13 smithi110 ceph-mon[86423]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T03:43:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:13 smithi110 ceph-mon[86423]: pgmap v222: 121 pgs: 121 active+clean; 491 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.9 MiB/s wr, 10 op/s 2024-09-14T03:43:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:13 smithi110 ceph-mon[88262]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T03:43:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:13 smithi110 ceph-mon[88262]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T03:43:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:13 smithi110 ceph-mon[88262]: pgmap v222: 121 pgs: 121 active+clean; 491 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.9 MiB/s wr, 10 op/s 2024-09-14T03:43:14.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:14.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:43:14.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:14.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:1650: done (4 left) 2024-09-14T03:43:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:1651: done (3 left) 2024-09-14T03:43:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:1652: done (2 left) 2024-09-14T03:43:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:1654: done (1 left) 2024-09-14T03:43:14.702 INFO:tasks.rados.rados.0.smithi110.stdout:1655: done (0 left) 2024-09-14T03:43:14.703 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 165 2024-09-14T03:43:14.703 INFO:tasks.rados.rados.0.smithi110.stdout:1656: seq_num 531 ranges {6447104=499712,6946816=417792,7364608=548864,7913472=237568} 2024-09-14T03:43:14.740 INFO:tasks.rados.rados.0.smithi110.stdout:1656: writing smithi11075562-44 from 6447104 to 6946816 tid 1 2024-09-14T03:43:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1656: writing smithi11075562-44 from 6946816 to 7364608 tid 2 2024-09-14T03:43:14.745 INFO:tasks.rados.rados.0.smithi110.stdout:1656: writing smithi11075562-44 from 7364608 to 7913472 tid 3 2024-09-14T03:43:14.747 INFO:tasks.rados.rados.0.smithi110.stdout:1656: writing smithi11075562-44 from 7913472 to 8151040 tid 4 2024-09-14T03:43:14.747 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 165 2024-09-14T03:43:14.747 INFO:tasks.rados.rados.0.smithi110.stdout:1657: seq_num 532 ranges {4685824=516096,5201920=696320,5898240=507904} 2024-09-14T03:43:14.775 INFO:tasks.rados.rados.0.smithi110.stdout:1657: writing smithi11075562-48 from 4685824 to 5201920 tid 1 2024-09-14T03:43:14.779 INFO:tasks.rados.rados.0.smithi110.stdout:1657: writing smithi11075562-48 from 5201920 to 5898240 tid 2 2024-09-14T03:43:14.783 INFO:tasks.rados.rados.0.smithi110.stdout:1657: writing smithi11075562-48 from 5898240 to 6406144 tid 3 2024-09-14T03:43:14.783 INFO:tasks.rados.rados.0.smithi110.stdout:1658: delete oid 25 current snap is 165 2024-09-14T03:43:14.783 INFO:tasks.rados.rados.0.smithi110.stdout:1656: finishing write tid 1 to smithi11075562-44 2024-09-14T03:43:14.783 INFO:tasks.rados.rados.0.smithi110.stdout:1656: finishing write tid 2 to smithi11075562-44 2024-09-14T03:43:14.783 INFO:tasks.rados.rados.0.smithi110.stdout:1656: finishing write tid 3 to smithi11075562-44 2024-09-14T03:43:14.783 INFO:tasks.rados.rados.0.smithi110.stdout:1656: finishing write tid 4 to smithi11075562-44 2024-09-14T03:43:14.783 INFO:tasks.rados.rados.0.smithi110.stdout:1656: finishing write tid 5 to smithi11075562-44 2024-09-14T03:43:14.784 INFO:tasks.rados.rados.0.smithi110.stdout:1656: finishing write tid 6 to smithi11075562-44 2024-09-14T03:43:14.784 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 393 (ObjNum 531 snap 165 seq_num 531) dirty exists 2024-09-14T03:43:14.784 INFO:tasks.rados.rados.0.smithi110.stdout:1656: left oid 44 (ObjNum 531 snap 165 seq_num 531) 2024-09-14T03:43:14.784 INFO:tasks.rados.rados.0.smithi110.stdout:1657: finishing write tid 1 to smithi11075562-48 2024-09-14T03:43:14.784 INFO:tasks.rados.rados.0.smithi110.stdout:1657: finishing write tid 2 to smithi11075562-48 2024-09-14T03:43:14.787 INFO:tasks.rados.rados.0.smithi110.stdout:1656: done (2 left) 2024-09-14T03:43:14.787 INFO:tasks.rados.rados.0.smithi110.stdout:1658: done (1 left) 2024-09-14T03:43:14.787 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 165 2024-09-14T03:43:14.787 INFO:tasks.rados.rados.0.smithi110.stdout:1657: finishing write tid 3 to smithi11075562-48 2024-09-14T03:43:14.787 INFO:tasks.rados.rados.0.smithi110.stdout:1659: seq_num 534 ranges {7700480=548864,8249344=516096,8765440=688128} 2024-09-14T03:43:14.830 INFO:tasks.rados.rados.0.smithi110.stdout:1659: writing smithi11075562-22 from 7700480 to 8249344 tid 1 2024-09-14T03:43:14.833 INFO:tasks.rados.rados.0.smithi110.stdout:1659: writing smithi11075562-22 from 8249344 to 8765440 tid 2 2024-09-14T03:43:14.837 INFO:tasks.rados.rados.0.smithi110.stdout:1659: writing smithi11075562-22 from 8765440 to 9453568 tid 3 2024-09-14T03:43:14.838 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 165 2024-09-14T03:43:14.838 INFO:tasks.rados.rados.0.smithi110.stdout:1657: finishing write tid 4 to smithi11075562-48 2024-09-14T03:43:14.838 INFO:tasks.rados.rados.0.smithi110.stdout:1660: seq_num 535 ranges {6561792=565248,7127040=802816,7929856=401408} 2024-09-14T03:43:14.874 INFO:tasks.rados.rados.0.smithi110.stdout:1660: writing smithi11075562-49 from 6561792 to 7127040 tid 1 2024-09-14T03:43:14.879 INFO:tasks.rados.rados.0.smithi110.stdout:1660: writing smithi11075562-49 from 7127040 to 7929856 tid 2 2024-09-14T03:43:14.882 INFO:tasks.rados.rados.0.smithi110.stdout:1660: writing smithi11075562-49 from 7929856 to 8331264 tid 3 2024-09-14T03:43:14.882 INFO:tasks.rados.rados.0.smithi110.stdout:1661: read oid 44 snap 129 2024-09-14T03:43:14.882 INFO:tasks.rados.rados.0.smithi110.stdout:1661: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:43:14.882 INFO:tasks.rados.rados.0.smithi110.stdout:1662: setattr oid 37 current snap is 165 2024-09-14T03:43:14.882 INFO:tasks.rados.rados.0.smithi110.stdout:1657: finishing write tid 5 to smithi11075562-48 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 907 (ObjNum 532 snap 165 seq_num 532) dirty exists 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:1657: left oid 48 (ObjNum 532 snap 165 seq_num 532) 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:1659: finishing write tid 1 to smithi11075562-22 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:1659: finishing write tid 2 to smithi11075562-22 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:1659: finishing write tid 3 to smithi11075562-22 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:1659: finishing write tid 4 to smithi11075562-22 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:1659: finishing write tid 5 to smithi11075562-22 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1031 (ObjNum 534 snap 165 seq_num 534) dirty exists 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:1659: left oid 22 (ObjNum 534 snap 165 seq_num 534) 2024-09-14T03:43:14.883 INFO:tasks.rados.rados.0.smithi110.stdout:1660: finishing write tid 1 to smithi11075562-49 2024-09-14T03:43:14.884 INFO:tasks.rados.rados.0.smithi110.stdout:1657: done (4 left) 2024-09-14T03:43:14.884 INFO:tasks.rados.rados.0.smithi110.stdout:1659: done (3 left) 2024-09-14T03:43:14.884 INFO:tasks.rados.rados.0.smithi110.stdout:1663: snap_remove snap 160 2024-09-14T03:43:15.708 INFO:tasks.rados.rados.0.smithi110.stdout:1660: finishing write tid 2 to smithi11075562-49 2024-09-14T03:43:15.708 INFO:tasks.rados.rados.0.smithi110.stdout:1660: finishing write tid 3 to smithi11075562-49 2024-09-14T03:43:15.708 INFO:tasks.rados.rados.0.smithi110.stdout:1660: finishing write tid 4 to smithi11075562-49 2024-09-14T03:43:15.708 INFO:tasks.rados.rados.0.smithi110.stdout:1660: finishing write tid 5 to smithi11075562-49 2024-09-14T03:43:15.708 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1032 (ObjNum 535 snap 165 seq_num 535) dirty exists 2024-09-14T03:43:15.708 INFO:tasks.rados.rados.0.smithi110.stdout:1660: left oid 49 (ObjNum 535 snap 165 seq_num 535) 2024-09-14T03:43:15.708 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1178 (ObjNum 460 snap 149 seq_num 460) dirty exists 2024-09-14T03:43:15.708 INFO:tasks.rados.rados.0.smithi110.stdout:1661: expect (ObjNum 313 snap 100 seq_num 313) 2024-09-14T03:43:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:1660: done (3 left) 2024-09-14T03:43:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:1661: done (2 left) 2024-09-14T03:43:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:1662: done (1 left) 2024-09-14T03:43:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:1663: done (0 left) 2024-09-14T03:43:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:1664: snap_remove snap 164 2024-09-14T03:43:15.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:15 smithi195 ceph-mon[82388]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T03:43:15.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:15.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:16.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:15 smithi110 ceph-mon[86423]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T03:43:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:15 smithi110 ceph-mon[88262]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T03:43:16.712 INFO:tasks.rados.rados.0.smithi110.stdout:1664: done (0 left) 2024-09-14T03:43:16.712 INFO:tasks.rados.rados.0.smithi110.stdout:1665: rmattr oid 39 current snap is 165 2024-09-14T03:43:16.712 INFO:tasks.rados.rados.0.smithi110.stdout:1665: done (0 left) 2024-09-14T03:43:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:1666: snap_create 2024-09-14T03:43:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:16 smithi110 ceph-mon[88262]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T03:43:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:16 smithi110 ceph-mon[88262]: pgmap v225: 121 pgs: 121 active+clean; 492 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.3 MiB/s wr, 12 op/s 2024-09-14T03:43:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:16 smithi110 ceph-mon[86423]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T03:43:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:16 smithi110 ceph-mon[86423]: pgmap v225: 121 pgs: 121 active+clean; 492 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.3 MiB/s wr, 12 op/s 2024-09-14T03:43:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:16 smithi195 ceph-mon[82388]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T03:43:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:16 smithi195 ceph-mon[82388]: pgmap v225: 121 pgs: 121 active+clean; 492 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.3 MiB/s wr, 12 op/s 2024-09-14T03:43:17.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:17.714 INFO:tasks.rados.rados.0.smithi110.stdout:1666: done (0 left) 2024-09-14T03:43:17.714 INFO:tasks.rados.rados.0.smithi110.stdout:1667: rollback oid 43 current snap is 166 2024-09-14T03:43:17.715 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 43 to 140 2024-09-14T03:43:17.715 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 166 2024-09-14T03:43:17.715 INFO:tasks.rados.rados.0.smithi110.stdout:1668: seq_num 536 ranges {0=581632,581632=688128,1269760=516096} 2024-09-14T03:43:17.717 INFO:tasks.rados.rados.0.smithi110.stdout:1668: writing smithi11075562-4 from 0 to 581632 tid 1 2024-09-14T03:43:17.722 INFO:tasks.rados.rados.0.smithi110.stdout:1668: writing smithi11075562-4 from 581632 to 1269760 tid 2 2024-09-14T03:43:17.726 INFO:tasks.rados.rados.0.smithi110.stdout:1668: writing smithi11075562-4 from 1269760 to 1785856 tid 3 2024-09-14T03:43:17.726 INFO:tasks.rados.rados.0.smithi110.stdout:1669: read oid 18 snap -1 2024-09-14T03:43:17.726 INFO:tasks.rados.rados.0.smithi110.stdout:1669: expect (ObjNum 528 snap 164 seq_num 528) 2024-09-14T03:43:17.726 INFO:tasks.rados.rados.0.smithi110.stdout:1670: copy_from oid 6 from oid 29 current snap is 166 2024-09-14T03:43:17.726 INFO:tasks.rados.rados.0.smithi110.stdout:1668: finishing write tid 1 to smithi11075562-4 2024-09-14T03:43:17.726 INFO:tasks.rados.rados.0.smithi110.stdout:1671: setattr oid 2 current snap is 166 2024-09-14T03:43:17.726 INFO:tasks.rados.rados.0.smithi110.stdout:1667: finishing rollback tid 0 to smithi11075562-43 2024-09-14T03:43:17.727 INFO:tasks.rados.rados.0.smithi110.stdout:1667: finishing rollback tid 1 to smithi11075562-43 2024-09-14T03:43:17.727 INFO:tasks.rados.rados.0.smithi110.stdout:1667: finishing rollback tid 2 to smithi11075562-43 2024-09-14T03:43:17.727 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 944 (ObjNum 413 snap 134 seq_num 413) dirty exists 2024-09-14T03:43:17.728 INFO:tasks.rados.rados.0.smithi110.stdout:1667: done (4 left) 2024-09-14T03:43:17.728 INFO:tasks.rados.rados.0.smithi110.stdout:1672: rollback oid 45 current snap is 166 2024-09-14T03:43:17.728 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 129 2024-09-14T03:43:17.729 INFO:tasks.rados.rados.0.smithi110.stdout:1673: read oid 33 snap -1 2024-09-14T03:43:17.729 INFO:tasks.rados.rados.0.smithi110.stdout:1673: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-14T03:43:17.729 INFO:tasks.rados.rados.0.smithi110.stdout:1674: snap_remove snap 133 2024-09-14T03:43:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:17 smithi110 ceph-mon[86423]: osdmap e400: 8 total, 8 up, 8 in 2024-09-14T03:43:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:17 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:17 smithi110 ceph-mon[88262]: osdmap e400: 8 total, 8 up, 8 in 2024-09-14T03:43:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:17 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:18.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:17 smithi195 ceph-mon[82388]: osdmap e400: 8 total, 8 up, 8 in 2024-09-14T03:43:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:17 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:18.699 INFO:tasks.rados.rados.0.smithi110.stdout:1674: done (6 left) 2024-09-14T03:43:18.699 INFO:tasks.rados.rados.0.smithi110.stdout:1675: read oid 48 snap -1 2024-09-14T03:43:18.699 INFO:tasks.rados.rados.0.smithi110.stdout:1675: expect (ObjNum 532 snap 165 seq_num 532) 2024-09-14T03:43:18.700 INFO:tasks.rados.rados.0.smithi110.stdout:1676: snap_remove snap 123 2024-09-14T03:43:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:18 smithi110 ceph-mon[88262]: osdmap e401: 8 total, 8 up, 8 in 2024-09-14T03:43:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:18 smithi110 ceph-mon[88262]: pgmap v228: 121 pgs: 121 active+clean; 499 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 21 op/s 2024-09-14T03:43:19.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:18 smithi110 ceph-mon[88262]: osdmap e402: 8 total, 8 up, 8 in 2024-09-14T03:43:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:18 smithi110 ceph-mon[86423]: osdmap e401: 8 total, 8 up, 8 in 2024-09-14T03:43:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:18 smithi110 ceph-mon[86423]: pgmap v228: 121 pgs: 121 active+clean; 499 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 21 op/s 2024-09-14T03:43:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:18 smithi110 ceph-mon[86423]: osdmap e402: 8 total, 8 up, 8 in 2024-09-14T03:43:19.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:18 smithi195 ceph-mon[82388]: osdmap e401: 8 total, 8 up, 8 in 2024-09-14T03:43:19.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:18 smithi195 ceph-mon[82388]: pgmap v228: 121 pgs: 121 active+clean; 499 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 21 op/s 2024-09-14T03:43:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:18 smithi195 ceph-mon[82388]: osdmap e402: 8 total, 8 up, 8 in 2024-09-14T03:43:19.701 INFO:tasks.rados.rados.0.smithi110.stdout:1676: done (7 left) 2024-09-14T03:43:19.701 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 166 2024-09-14T03:43:19.701 INFO:tasks.rados.rados.0.smithi110.stdout:1677: seq_num 537 ranges {2072576=598016,2670592=573440,3244032=491520,3735552=139264} 2024-09-14T03:43:19.717 INFO:tasks.rados.rados.0.smithi110.stdout:1677: writing smithi11075562-9 from 2072576 to 2670592 tid 1 2024-09-14T03:43:19.721 INFO:tasks.rados.rados.0.smithi110.stdout:1677: writing smithi11075562-9 from 2670592 to 3244032 tid 2 2024-09-14T03:43:19.724 INFO:tasks.rados.rados.0.smithi110.stdout:1677: writing smithi11075562-9 from 3244032 to 3735552 tid 3 2024-09-14T03:43:19.725 INFO:tasks.rados.rados.0.smithi110.stdout:1677: writing smithi11075562-9 from 3735552 to 3874816 tid 4 2024-09-14T03:43:19.725 INFO:tasks.rados.rados.0.smithi110.stdout:1678: snap_remove snap 130 2024-09-14T03:43:20.702 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:20.710 INFO:tasks.rados.rados.0.smithi110.stdout:1678: done (8 left) 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 166 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1668: finishing write tid 2 to smithi11075562-4 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1668: finishing write tid 3 to smithi11075562-4 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1668: finishing write tid 4 to smithi11075562-4 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 692 (ObjNum 455 snap 147 seq_num 455) dirty exists 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1670: finishing copy_from to smithi11075562-6 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1670: got expected ENOENT (src dne) 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1670: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1668: finishing write tid 5 to smithi11075562-4 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 903 (ObjNum 536 snap 166 seq_num 536) dirty exists 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1668: left oid 4 (ObjNum 536 snap 166 seq_num 536) 2024-09-14T03:43:20.711 INFO:tasks.rados.rados.0.smithi110.stdout:1672: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:43:20.712 INFO:tasks.rados.rados.0.smithi110.stdout:1672: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:43:20.712 INFO:tasks.rados.rados.0.smithi110.stdout:1672: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:43:20.712 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 1192 (ObjNum 376 snap 123 seq_num 376) dirty exists 2024-09-14T03:43:20.712 INFO:tasks.rados.rados.0.smithi110.stdout:1669: expect (ObjNum 528 snap 164 seq_num 528) 2024-09-14T03:43:20.759 INFO:tasks.rados.rados.0.smithi110.stdout:1673: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-14T03:43:20.938 INFO:tasks.rados.rados.0.smithi110.stdout:1675: expect (ObjNum 532 snap 165 seq_num 532) 2024-09-14T03:43:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:20 smithi195 ceph-mon[82388]: osdmap e403: 8 total, 8 up, 8 in 2024-09-14T03:43:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:20 smithi195 ceph-mon[82388]: pgmap v231: 121 pgs: 121 active+clean; 495 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-09-14T03:43:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:20 smithi110 ceph-mon[88262]: osdmap e403: 8 total, 8 up, 8 in 2024-09-14T03:43:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:20 smithi110 ceph-mon[88262]: pgmap v231: 121 pgs: 121 active+clean; 495 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-09-14T03:43:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:20 smithi110 ceph-mon[86423]: osdmap e403: 8 total, 8 up, 8 in 2024-09-14T03:43:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:20 smithi110 ceph-mon[86423]: pgmap v231: 121 pgs: 121 active+clean; 495 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-09-14T03:43:21.324 INFO:tasks.rados.rados.0.smithi110.stdout:1677: finishing write tid 1 to smithi11075562-9 2024-09-14T03:43:21.324 INFO:tasks.rados.rados.0.smithi110.stdout:1677: finishing write tid 2 to smithi11075562-9 2024-09-14T03:43:21.324 INFO:tasks.rados.rados.0.smithi110.stdout:1677: finishing write tid 3 to smithi11075562-9 2024-09-14T03:43:21.324 INFO:tasks.rados.rados.0.smithi110.stdout:1677: finishing write tid 4 to smithi11075562-9 2024-09-14T03:43:21.324 INFO:tasks.rados.rados.0.smithi110.stdout:1677: finishing write tid 5 to smithi11075562-9 2024-09-14T03:43:21.325 INFO:tasks.rados.rados.0.smithi110.stdout:1677: finishing write tid 6 to smithi11075562-9 2024-09-14T03:43:21.325 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1024 (ObjNum 537 snap 166 seq_num 537) dirty exists 2024-09-14T03:43:21.325 INFO:tasks.rados.rados.0.smithi110.stdout:1677: left oid 9 (ObjNum 537 snap 166 seq_num 537) 2024-09-14T03:43:21.325 INFO:tasks.rados.rados.0.smithi110.stdout:1679: seq_num 538 ranges {999424=614400,1613824=450560,2064384=704512,2768896=49152} 2024-09-14T03:43:21.332 INFO:tasks.rados.rados.0.smithi110.stdout:1679: writing smithi11075562-15 from 999424 to 1613824 tid 1 2024-09-14T03:43:21.335 INFO:tasks.rados.rados.0.smithi110.stdout:1679: writing smithi11075562-15 from 1613824 to 2064384 tid 2 2024-09-14T03:43:21.340 INFO:tasks.rados.rados.0.smithi110.stdout:1679: writing smithi11075562-15 from 2064384 to 2768896 tid 3 2024-09-14T03:43:21.340 INFO:tasks.rados.rados.0.smithi110.stdout:1679: writing smithi11075562-15 from 2768896 to 2818048 tid 4 2024-09-14T03:43:21.340 INFO:tasks.rados.rados.0.smithi110.stdout:1668: done (8 left) 2024-09-14T03:43:21.340 INFO:tasks.rados.rados.0.smithi110.stdout:1669: done (7 left) 2024-09-14T03:43:21.340 INFO:tasks.rados.rados.0.smithi110.stdout:1670: done (6 left) 2024-09-14T03:43:21.340 INFO:tasks.rados.rados.0.smithi110.stdout:1671: done (5 left) 2024-09-14T03:43:21.340 INFO:tasks.rados.rados.0.smithi110.stdout:1672: done (4 left) 2024-09-14T03:43:21.340 INFO:tasks.rados.rados.0.smithi110.stdout:1673: done (3 left) 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:1675: done (2 left) 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:1677: done (1 left) 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:1680: copy_from oid 45 from oid 38 current snap is 166 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:1679: finishing write tid 1 to smithi11075562-15 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:1679: finishing write tid 2 to smithi11075562-15 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:1681: rollback oid 31 current snap is 166 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 129 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:1682: rollback oid 17 current snap is 166 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 140 2024-09-14T03:43:21.341 INFO:tasks.rados.rados.0.smithi110.stdout:1683: snap_remove snap 90 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1683: done (4 left) 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1684: read oid 48 snap 145 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1684: expect (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1685: read oid 25 snap -1 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1685: expect deleted 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 166 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1679: finishing write tid 3 to smithi11075562-15 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1681: finishing rollback tid 0 to smithi11075562-31 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1679: finishing write tid 4 to smithi11075562-15 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1679: finishing write tid 5 to smithi11075562-15 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1682: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:43:21.709 INFO:tasks.rados.rados.0.smithi110.stdout:1686: seq_num 539 ranges {4825088=630784,5455872=737280,6193152=466944} 2024-09-14T03:43:21.742 INFO:tasks.rados.rados.0.smithi110.stdout:1686: writing smithi11075562-12 from 4825088 to 5455872 tid 1 2024-09-14T03:43:21.746 INFO:tasks.rados.rados.0.smithi110.stdout:1686: writing smithi11075562-12 from 5455872 to 6193152 tid 2 2024-09-14T03:43:21.749 INFO:tasks.rados.rados.0.smithi110.stdout:1686: writing smithi11075562-12 from 6193152 to 6660096 tid 3 2024-09-14T03:43:21.749 INFO:tasks.rados.rados.0.smithi110.stdout:1687: snap_remove snap 165 2024-09-14T03:43:21.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:21 smithi110 ceph-mon[88262]: osdmap e404: 8 total, 8 up, 8 in 2024-09-14T03:43:21.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:21 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:43:21.891 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:43:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:43:21] "GET /metrics HTTP/1.1" 200 36273 "" "Prometheus/2.51.0" 2024-09-14T03:43:21.891 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:21 smithi110 ceph-mon[86423]: osdmap e404: 8 total, 8 up, 8 in 2024-09-14T03:43:21.891 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:21 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:43:21.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:21 smithi195 ceph-mon[82388]: osdmap e404: 8 total, 8 up, 8 in 2024-09-14T03:43:21.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:21 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:43:22.715 INFO:tasks.rados.rados.0.smithi110.stdout:1687: done (7 left) 2024-09-14T03:43:22.715 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 166 2024-09-14T03:43:22.715 INFO:tasks.rados.rados.0.smithi110.stdout:1681: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:43:22.715 INFO:tasks.rados.rados.0.smithi110.stdout:1682: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:43:22.715 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 909 (ObjNum 129 snap 0 seq_num 65241768) dirty dne 2024-09-14T03:43:22.715 INFO:tasks.rados.rados.0.smithi110.stdout:1681: finishing rollback tid 2 to smithi11075562-31 2024-09-14T03:43:22.715 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 906 (ObjNum 357 snap 115 seq_num 357) dirty exists 2024-09-14T03:43:22.716 INFO:tasks.rados.rados.0.smithi110.stdout:1679: finishing write tid 6 to smithi11075562-15 2024-09-14T03:43:22.716 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1063 (ObjNum 538 snap 166 seq_num 538) dirty exists 2024-09-14T03:43:22.716 INFO:tasks.rados.rados.0.smithi110.stdout:1679: left oid 15 (ObjNum 538 snap 166 seq_num 538) 2024-09-14T03:43:22.716 INFO:tasks.rados.rados.0.smithi110.stdout:1680: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:43:22.716 INFO:tasks.rados.rados.0.smithi110.stdout:1680: finishing copy_from to smithi11075562-45 2024-09-14T03:43:22.716 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 527 (ObjNum 519 snap 163 seq_num 519) dirty exists 2024-09-14T03:43:22.716 INFO:tasks.rados.rados.0.smithi110.stdout:1684: expect (ObjNum 329 snap 105 seq_num 329) 2024-09-14T03:43:23.003 INFO:tasks.rados.rados.0.smithi110.stdout:1688: seq_num 540 ranges {2654208=647168,3301376=671744,3973120=532480} 2024-09-14T03:43:23.021 INFO:tasks.rados.rados.0.smithi110.stdout:1688: writing smithi11075562-16 from 2654208 to 3301376 tid 1 2024-09-14T03:43:23.024 INFO:tasks.rados.rados.0.smithi110.stdout:1688: writing smithi11075562-16 from 3301376 to 3973120 tid 2 2024-09-14T03:43:23.027 INFO:tasks.rados.rados.0.smithi110.stdout:1688: writing smithi11075562-16 from 3973120 to 4505600 tid 3 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1679: done (7 left) 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1680: done (6 left) 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1681: done (5 left) 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1682: done (4 left) 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1684: done (3 left) 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1685: done (2 left) 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 166 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1686: finishing write tid 1 to smithi11075562-12 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1686: finishing write tid 2 to smithi11075562-12 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1686: finishing write tid 3 to smithi11075562-12 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1686: finishing write tid 4 to smithi11075562-12 2024-09-14T03:43:23.028 INFO:tasks.rados.rados.0.smithi110.stdout:1689: seq_num 541 ranges {0=663552,663552=557056,1220608=475136,1695744=172032} 2024-09-14T03:43:23.031 INFO:tasks.rados.rados.0.smithi110.stdout:1689: writing smithi11075562-23 from 0 to 663552 tid 1 2024-09-14T03:43:23.034 INFO:tasks.rados.rados.0.smithi110.stdout:1689: writing smithi11075562-23 from 663552 to 1220608 tid 2 2024-09-14T03:43:23.037 INFO:tasks.rados.rados.0.smithi110.stdout:1689: writing smithi11075562-23 from 1220608 to 1695744 tid 3 2024-09-14T03:43:23.038 INFO:tasks.rados.rados.0.smithi110.stdout:1689: writing smithi11075562-23 from 1695744 to 1867776 tid 4 2024-09-14T03:43:23.038 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 166 2024-09-14T03:43:23.038 INFO:tasks.rados.rados.0.smithi110.stdout:1686: finishing write tid 5 to smithi11075562-12 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1236 (ObjNum 539 snap 166 seq_num 539) dirty exists 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1686: left oid 12 (ObjNum 539 snap 166 seq_num 539) 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1688: finishing write tid 1 to smithi11075562-16 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1688: finishing write tid 2 to smithi11075562-16 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1688: finishing write tid 3 to smithi11075562-16 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1688: finishing write tid 4 to smithi11075562-16 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1688: finishing write tid 5 to smithi11075562-16 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 981 (ObjNum 540 snap 166 seq_num 540) dirty exists 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1688: left oid 16 (ObjNum 540 snap 166 seq_num 540) 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1689: finishing write tid 1 to smithi11075562-23 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1689: finishing write tid 2 to smithi11075562-23 2024-09-14T03:43:23.039 INFO:tasks.rados.rados.0.smithi110.stdout:1690: seq_num 542 ranges {1818624=679936,2498560=442368,2940928=679936,3620864=81920} 2024-09-14T03:43:23.052 INFO:tasks.rados.rados.0.smithi110.stdout:1690: writing smithi11075562-5 from 1818624 to 2498560 tid 1 2024-09-14T03:43:23.054 INFO:tasks.rados.rados.0.smithi110.stdout:1690: writing smithi11075562-5 from 2498560 to 2940928 tid 2 2024-09-14T03:43:23.058 INFO:tasks.rados.rados.0.smithi110.stdout:1690: writing smithi11075562-5 from 2940928 to 3620864 tid 3 2024-09-14T03:43:23.059 INFO:tasks.rados.rados.0.smithi110.stdout:1690: writing smithi11075562-5 from 3620864 to 3702784 tid 4 2024-09-14T03:43:23.059 INFO:tasks.rados.rados.0.smithi110.stdout:1686: done (3 left) 2024-09-14T03:43:23.059 INFO:tasks.rados.rados.0.smithi110.stdout:1688: done (2 left) 2024-09-14T03:43:23.059 INFO:tasks.rados.rados.0.smithi110.stdout:1691: rollback oid 25 current snap is 166 2024-09-14T03:43:23.059 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 25 to 154 2024-09-14T03:43:23.059 INFO:tasks.rados.rados.0.smithi110.stdout:1689: finishing write tid 3 to smithi11075562-23 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:1689: finishing write tid 4 to smithi11075562-23 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:1689: finishing write tid 5 to smithi11075562-23 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:1689: finishing write tid 6 to smithi11075562-23 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 906 (ObjNum 541 snap 166 seq_num 541) dirty exists 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:1689: left oid 23 (ObjNum 541 snap 166 seq_num 541) 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:1690: finishing write tid 1 to smithi11075562-5 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:1690: finishing write tid 2 to smithi11075562-5 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:1689: done (2 left) 2024-09-14T03:43:23.060 INFO:tasks.rados.rados.0.smithi110.stdout:1692: snap_remove snap 146 2024-09-14T03:43:23.106 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:22 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:43:23.106 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:22 smithi110 ceph-mon[88262]: Upgrade: osd.3 is safe to restart 2024-09-14T03:43:23.106 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:22 smithi110 ceph-mon[88262]: osdmap e405: 8 total, 8 up, 8 in 2024-09-14T03:43:23.106 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:22 smithi110 ceph-mon[88262]: pgmap v234: 121 pgs: 121 active+clean; 482 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 864 KiB/s wr, 7 op/s 2024-09-14T03:43:23.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:22 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:43:23.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:22 smithi110 ceph-mon[86423]: Upgrade: osd.3 is safe to restart 2024-09-14T03:43:23.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:22 smithi110 ceph-mon[86423]: osdmap e405: 8 total, 8 up, 8 in 2024-09-14T03:43:23.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:22 smithi110 ceph-mon[86423]: pgmap v234: 121 pgs: 121 active+clean; 482 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 864 KiB/s wr, 7 op/s 2024-09-14T03:43:23.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:22 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:43:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:22 smithi195 ceph-mon[82388]: Upgrade: osd.3 is safe to restart 2024-09-14T03:43:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:22 smithi195 ceph-mon[82388]: osdmap e405: 8 total, 8 up, 8 in 2024-09-14T03:43:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:22 smithi195 ceph-mon[82388]: pgmap v234: 121 pgs: 121 active+clean; 482 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 864 KiB/s wr, 7 op/s 2024-09-14T03:43:23.699 INFO:tasks.rados.rados.0.smithi110.stdout:1692: done (2 left) 2024-09-14T03:43:23.699 INFO:tasks.rados.rados.0.smithi110.stdout:1693: snap_remove snap 129 2024-09-14T03:43:24.002 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:43:24.003 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[88262]: osdmap e406: 8 total, 8 up, 8 in 2024-09-14T03:43:24.003 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[88262]: Upgrade: Updating osd.3 2024-09-14T03:43:24.003 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:24.003 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:43:24.003 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:24.003 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[88262]: Deploying daemon osd.3 on smithi110 2024-09-14T03:43:24.003 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[88262]: osdmap e407: 8 total, 8 up, 8 in 2024-09-14T03:43:24.003 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[86423]: osdmap e406: 8 total, 8 up, 8 in 2024-09-14T03:43:24.004 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[86423]: Upgrade: Updating osd.3 2024-09-14T03:43:24.004 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:24.004 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:43:24.004 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:24.004 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[86423]: Deploying daemon osd.3 on smithi110 2024-09-14T03:43:24.004 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:23 smithi110 ceph-mon[86423]: osdmap e407: 8 total, 8 up, 8 in 2024-09-14T03:43:24.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:23 smithi195 ceph-mon[82388]: osdmap e406: 8 total, 8 up, 8 in 2024-09-14T03:43:24.150 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:23 smithi195 ceph-mon[82388]: Upgrade: Updating osd.3 2024-09-14T03:43:24.150 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:23 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:24.150 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:23 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:43:24.150 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:23 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:24.150 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:23 smithi195 ceph-mon[82388]: Deploying daemon osd.3 on smithi110 2024-09-14T03:43:24.150 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:23 smithi195 ceph-mon[82388]: osdmap e407: 8 total, 8 up, 8 in 2024-09-14T03:43:24.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:24 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:24.579 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:43:24.579 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (4m) 21s ago 8m 20.3M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:43:24.579 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (3m) 2m ago 8m 78.7M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:43:24.579 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (9m) 21s ago 9m 20.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:43:24.579 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (9m) 2m ago 9m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:43:24.579 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (3m) 2m ago 12m 452M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:43:24.579 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (3m) 21s ago 14m 524M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:43:24.579 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (2m) 21s ago 14m 52.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (2m) 2m ago 13m 21.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (2m) 21s ago 13m 45.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (4m) 21s ago 7m 10.8M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (3m) 2m ago 7m 9722k - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (91s) 21s ago 12m 185M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (58s) 21s ago 12m 90.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (25s) 21s ago 11m 11.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 0cdc58c38711 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (11m) 21s ago 11m 163M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 efe1d43f93e1 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (11m) 2m ago 11m 229M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (10m) 2m ago 10m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (10m) 2m ago 10m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (9m) 2m ago 9m 106M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:43:24.580 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (3m) 2m ago 8m 44.2M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1690: finishing write tid 3 to smithi11075562-5 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1690: finishing write tid 4 to smithi11075562-5 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1690: finishing write tid 5 to smithi11075562-5 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1690: finishing write tid 6 to smithi11075562-5 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1246 (ObjNum 542 snap 166 seq_num 542) dirty exists 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1690: left oid 5 (ObjNum 542 snap 166 seq_num 542) 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1691: finishing rollback tid 1 to smithi11075562-25 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1691: finishing rollback tid 2 to smithi11075562-25 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1248 (ObjNum 420 snap 136 seq_num 420) dirty exists 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1690: done (2 left) 2024-09-14T03:43:24.703 INFO:tasks.rados.rados.0.smithi110.stdout:1691: done (1 left) 2024-09-14T03:43:24.704 INFO:tasks.rados.rados.0.smithi110.stdout:1693: done (0 left) 2024-09-14T03:43:24.704 INFO:tasks.rados.rados.0.smithi110.stdout:1694: read oid 50 snap 154 2024-09-14T03:43:24.704 INFO:tasks.rados.rados.0.smithi110.stdout:1694: expect deleted 2024-09-14T03:43:24.704 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 166 2024-09-14T03:43:24.704 INFO:tasks.rados.rados.0.smithi110.stdout:1695: seq_num 543 ranges {3874816=704512,4579328=729088,5308416=475136} 2024-09-14T03:43:24.730 INFO:tasks.rados.rados.0.smithi110.stdout:1695: writing smithi11075562-9 from 3874816 to 4579328 tid 1 2024-09-14T03:43:24.735 INFO:tasks.rados.rados.0.smithi110.stdout:1695: writing smithi11075562-9 from 4579328 to 5308416 tid 2 2024-09-14T03:43:24.738 INFO:tasks.rados.rados.0.smithi110.stdout:1695: writing smithi11075562-9 from 5308416 to 5783552 tid 3 2024-09-14T03:43:24.738 INFO:tasks.rados.rados.0.smithi110.stdout:1696: delete oid 1 current snap is 166 2024-09-14T03:43:24.738 INFO:tasks.rados.rados.0.smithi110.stdout:1695: finishing write tid 1 to smithi11075562-9 2024-09-14T03:43:24.738 INFO:tasks.rados.rados.0.smithi110.stdout:1695: finishing write tid 2 to smithi11075562-9 2024-09-14T03:43:24.741 INFO:tasks.rados.rados.0.smithi110.stdout:1694: done (2 left) 2024-09-14T03:43:24.741 INFO:tasks.rados.rados.0.smithi110.stdout:1696: done (1 left) 2024-09-14T03:43:24.741 INFO:tasks.rados.rados.0.smithi110.stdout:1697: read oid 15 snap -1 2024-09-14T03:43:24.741 INFO:tasks.rados.rados.0.smithi110.stdout:1697: expect (ObjNum 538 snap 166 seq_num 538) 2024-09-14T03:43:24.741 INFO:tasks.rados.rados.0.smithi110.stdout:1698: copy_from oid 8 from oid 19 current snap is 166 2024-09-14T03:43:24.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 166 2024-09-14T03:43:24.741 INFO:tasks.rados.rados.0.smithi110.stdout:1699: seq_num 545 ranges {2064384=737280,2801664=491520,3293184=450560,3743744=262144} 2024-09-14T03:43:24.756 INFO:tasks.rados.rados.0.smithi110.stdout:1699: writing smithi11075562-37 from 2064384 to 2801664 tid 1 2024-09-14T03:43:24.759 INFO:tasks.rados.rados.0.smithi110.stdout:1699: writing smithi11075562-37 from 2801664 to 3293184 tid 2 2024-09-14T03:43:24.762 INFO:tasks.rados.rados.0.smithi110.stdout:1699: writing smithi11075562-37 from 3293184 to 3743744 tid 3 2024-09-14T03:43:24.764 INFO:tasks.rados.rados.0.smithi110.stdout:1699: writing smithi11075562-37 from 3743744 to 4005888 tid 4 2024-09-14T03:43:24.764 INFO:tasks.rados.rados.0.smithi110.stdout:1700: copy_from oid 47 from oid 49 current snap is 166 2024-09-14T03:43:24.764 INFO:tasks.rados.rados.0.smithi110.stdout:1695: finishing write tid 3 to smithi11075562-9 2024-09-14T03:43:24.764 INFO:tasks.rados.rados.0.smithi110.stdout:1695: finishing write tid 4 to smithi11075562-9 2024-09-14T03:43:24.764 INFO:tasks.rados.rados.0.smithi110.stdout:1695: finishing write tid 5 to smithi11075562-9 2024-09-14T03:43:24.764 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1028 (ObjNum 543 snap 166 seq_num 543) dirty exists 2024-09-14T03:43:24.764 INFO:tasks.rados.rados.0.smithi110.stdout:1695: left oid 9 (ObjNum 543 snap 166 seq_num 543) 2024-09-14T03:43:24.764 INFO:tasks.rados.rados.0.smithi110.stdout:1697: expect (ObjNum 538 snap 166 seq_num 538) 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1698: finishing copy_from to smithi11075562-8 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1698: got expected ENOENT (src dne) 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1698: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1699: finishing write tid 1 to smithi11075562-37 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1699: finishing write tid 2 to smithi11075562-37 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1699: finishing write tid 3 to smithi11075562-37 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1699: finishing write tid 4 to smithi11075562-37 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1699: finishing write tid 5 to smithi11075562-37 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1699: finishing write tid 6 to smithi11075562-37 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1277 (ObjNum 545 snap 166 seq_num 545) dirty exists 2024-09-14T03:43:24.855 INFO:tasks.rados.rados.0.smithi110.stdout:1699: left oid 37 (ObjNum 545 snap 166 seq_num 545) 2024-09-14T03:43:24.856 INFO:tasks.rados.rados.0.smithi110.stdout:1700: finishing copy_from racing read to smithi11075562-47 2024-09-14T03:43:24.856 INFO:tasks.rados.rados.0.smithi110.stdout:1695: done (4 left) 2024-09-14T03:43:24.856 INFO:tasks.rados.rados.0.smithi110.stdout:1697: done (3 left) 2024-09-14T03:43:24.856 INFO:tasks.rados.rados.0.smithi110.stdout:1698: done (2 left) 2024-09-14T03:43:24.856 INFO:tasks.rados.rados.0.smithi110.stdout:1699: done (1 left) 2024-09-14T03:43:24.856 INFO:tasks.rados.rados.0.smithi110.stdout:1701: read oid 34 snap -1 2024-09-14T03:43:24.856 INFO:tasks.rados.rados.0.smithi110.stdout:1701: expect deleted 2024-09-14T03:43:24.856 INFO:tasks.rados.rados.0.smithi110.stdout:1702: snap_remove snap 154 2024-09-14T03:43:24.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:24 smithi110 ceph-mon[88262]: pgmap v237: 121 pgs: 121 active+clean; 416 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 10 op/s 2024-09-14T03:43:24.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:24 smithi110 ceph-mon[88262]: osdmap e408: 8 total, 8 up, 8 in 2024-09-14T03:43:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:24 smithi110 ceph-mon[86423]: pgmap v237: 121 pgs: 121 active+clean; 416 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 10 op/s 2024-09-14T03:43:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:24 smithi110 ceph-mon[86423]: osdmap e408: 8 total, 8 up, 8 in 2024-09-14T03:43:24.948 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:43:24.949 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:43:24.949 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:43:24.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:43:24.949 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:43:24.949 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 5, 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:43:24.950 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:43:24.951 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 7, 2024-09-14T03:43:24.951 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:43:24.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:43:24.951 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:43:25.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:24 smithi195 ceph-mon[82388]: pgmap v237: 121 pgs: 121 active+clean; 416 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 10 op/s 2024-09-14T03:43:25.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:24 smithi195 ceph-mon[82388]: osdmap e408: 8 total, 8 up, 8 in 2024-09-14T03:43:25.235 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:43:25.235 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:43:25.235 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:43:25.235 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:43:25.235 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:43:25.235 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:43:25.236 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:43:25.236 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:43:25.236 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "8/20 daemons upgraded", 2024-09-14T03:43:25.236 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:43:25.236 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:43:25.236 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:43:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:1702: done (2 left) 2024-09-14T03:43:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:1703: snap_create 2024-09-14T03:43:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:1700: finishing copy_from to smithi11075562-47 2024-09-14T03:43:25.731 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1032 (ObjNum 535 snap 165 seq_num 535) dirty exists 2024-09-14T03:43:25.870 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:25 smithi110 systemd[1]: Stopping Ceph osd.3 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:43:25.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-mon[88262]: from='client.44246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:25.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-mon[88262]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:25.871 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-mon[88262]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:25.871 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3200033876' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:25.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-mon[86423]: from='client.44246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:25.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-mon[86423]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:25.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-mon[86423]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:25.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3200033876' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:26.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[56794]: 2024-09-14T03:43:25.868+0000 7f6312aba640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:43:26.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[56794]: 2024-09-14T03:43:25.868+0000 7f6312aba640 -1 osd.3 409 *** Got signal Terminated *** 2024-09-14T03:43:26.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:25 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[56794]: 2024-09-14T03:43:25.868+0000 7f6312aba640 -1 osd.3 409 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:43:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:25 smithi195 ceph-mon[82388]: from='client.44246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:25 smithi195 ceph-mon[82388]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:25 smithi195 ceph-mon[82388]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:25 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3200033876' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:26.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:26.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:26.740 INFO:tasks.rados.rados.0.smithi110.stdout:1700: done (2 left) 2024-09-14T03:43:26.741 INFO:tasks.rados.rados.0.smithi110.stdout:1701: done (1 left) 2024-09-14T03:43:26.741 INFO:tasks.rados.rados.0.smithi110.stdout:1703: done (0 left) 2024-09-14T03:43:26.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 167 2024-09-14T03:43:26.741 INFO:tasks.rados.rados.0.smithi110.stdout:1704: seq_num 546 ranges {0=753664,753664=778240,1531904=425984} 2024-09-14T03:43:26.745 INFO:tasks.rados.rados.0.smithi110.stdout:1704: writing smithi11075562-29 from 0 to 753664 tid 1 2024-09-14T03:43:26.750 INFO:tasks.rados.rados.0.smithi110.stdout:1704: writing smithi11075562-29 from 753664 to 1531904 tid 2 2024-09-14T03:43:26.752 INFO:tasks.rados.rados.0.smithi110.stdout:1704: writing smithi11075562-29 from 1531904 to 1957888 tid 3 2024-09-14T03:43:26.752 INFO:tasks.rados.rados.0.smithi110.stdout:1705: snap_remove snap 145 2024-09-14T03:43:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:26 smithi110 ceph-mon[88262]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:26 smithi110 ceph-mon[88262]: osdmap e409: 8 total, 8 up, 8 in 2024-09-14T03:43:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:26 smithi110 ceph-mon[88262]: pgmap v240: 121 pgs: 121 active+clean; 364 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.8 MiB/s wr, 12 op/s 2024-09-14T03:43:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:26 smithi110 ceph-mon[88262]: osd.3 marked itself down and dead 2024-09-14T03:43:27.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:26 smithi110 ceph-mon[86423]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:27.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:26 smithi110 ceph-mon[86423]: osdmap e409: 8 total, 8 up, 8 in 2024-09-14T03:43:27.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:26 smithi110 ceph-mon[86423]: pgmap v240: 121 pgs: 121 active+clean; 364 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.8 MiB/s wr, 12 op/s 2024-09-14T03:43:27.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:26 smithi110 ceph-mon[86423]: osd.3 marked itself down and dead 2024-09-14T03:43:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:26 smithi195 ceph-mon[82388]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:26 smithi195 ceph-mon[82388]: osdmap e409: 8 total, 8 up, 8 in 2024-09-14T03:43:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:26 smithi195 ceph-mon[82388]: pgmap v240: 121 pgs: 121 active+clean; 364 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.8 MiB/s wr, 12 op/s 2024-09-14T03:43:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:26 smithi195 ceph-mon[82388]: osd.3 marked itself down and dead 2024-09-14T03:43:27.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:26 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:27.570 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:27 smithi110 podman[106508]: 2024-09-14 03:43:27.28229271 +0000 UTC m=+1.514435574 container stop efe1d43f93e1e5defd740e373e2fa178fcd956930044cc2fd927058dffec3b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:43:27.570 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:27 smithi110 podman[106508]: 2024-09-14 03:43:27.308007276 +0000 UTC m=+1.540150140 container died efe1d43f93e1e5defd740e373e2fa178fcd956930044cc2fd927058dffec3b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:43:27.737 INFO:tasks.rados.rados.0.smithi110.stdout:1705: done (1 left) 2024-09-14T03:43:27.737 INFO:tasks.rados.rados.0.smithi110.stdout:1706: rmattr oid 40 current snap is 167 2024-09-14T03:43:27.737 INFO:tasks.rados.rados.0.smithi110.stdout:1706: done (1 left) 2024-09-14T03:43:27.737 INFO:tasks.rados.rados.0.smithi110.stdout:1707: delete oid 21 current snap is 167 2024-09-14T03:43:27.756 INFO:tasks.rados.rados.0.smithi110.stdout:1707: done (1 left) 2024-09-14T03:43:27.756 INFO:tasks.rados.rados.0.smithi110.stdout:1708: read oid 44 snap -1 2024-09-14T03:43:27.756 INFO:tasks.rados.rados.0.smithi110.stdout:1708: expect (ObjNum 531 snap 165 seq_num 531) 2024-09-14T03:43:27.757 INFO:tasks.rados.rados.0.smithi110.stdout:1709: delete oid 41 current snap is 167 2024-09-14T03:43:27.759 INFO:tasks.rados.rados.0.smithi110.stdout:1704: finishing write tid 1 to smithi11075562-29 2024-09-14T03:43:27.759 INFO:tasks.rados.rados.0.smithi110.stdout:1704: finishing write tid 2 to smithi11075562-29 2024-09-14T03:43:27.760 INFO:tasks.rados.rados.0.smithi110.stdout:1704: finishing write tid 3 to smithi11075562-29 2024-09-14T03:43:27.760 INFO:tasks.rados.rados.0.smithi110.stdout:1704: finishing write tid 4 to smithi11075562-29 2024-09-14T03:43:27.760 INFO:tasks.rados.rados.0.smithi110.stdout:1709: done (2 left) 2024-09-14T03:43:27.760 INFO:tasks.rados.rados.0.smithi110.stdout:1710: delete oid 47 current snap is 167 2024-09-14T03:43:27.763 INFO:tasks.rados.rados.0.smithi110.stdout:1704: finishing write tid 5 to smithi11075562-29 2024-09-14T03:43:27.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 573 (ObjNum 546 snap 167 seq_num 546) dirty exists 2024-09-14T03:43:27.763 INFO:tasks.rados.rados.0.smithi110.stdout:1704: left oid 29 (ObjNum 546 snap 167 seq_num 546) 2024-09-14T03:43:27.768 INFO:tasks.rados.rados.0.smithi110.stdout:1704: done (2 left) 2024-09-14T03:43:27.768 INFO:tasks.rados.rados.0.smithi110.stdout:1710: done (1 left) 2024-09-14T03:43:27.768 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 167 2024-09-14T03:43:27.768 INFO:tasks.rados.rados.0.smithi110.stdout:1711: seq_num 550 ranges {3448832=417792,3866624=770048,4636672=638976,5275648=196608} 2024-09-14T03:43:27.789 INFO:tasks.rados.rados.0.smithi110.stdout:1711: writing smithi11075562-33 from 3448832 to 3866624 tid 1 2024-09-14T03:43:27.794 INFO:tasks.rados.rados.0.smithi110.stdout:1711: writing smithi11075562-33 from 3866624 to 4636672 tid 2 2024-09-14T03:43:27.797 INFO:tasks.rados.rados.0.smithi110.stdout:1711: writing smithi11075562-33 from 4636672 to 5275648 tid 3 2024-09-14T03:43:27.799 INFO:tasks.rados.rados.0.smithi110.stdout:1711: writing smithi11075562-33 from 5275648 to 5472256 tid 4 2024-09-14T03:43:27.799 INFO:tasks.rados.rados.0.smithi110.stdout:1712: rollback oid 6 current snap is 167 2024-09-14T03:43:27.799 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 6 to 166 2024-09-14T03:43:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:1713: read oid 8 snap 140 2024-09-14T03:43:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:1713: expect (ObjNum 418 snap 136 seq_num 418) 2024-09-14T03:43:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:1714: rollback oid 17 current snap is 167 2024-09-14T03:43:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 166 2024-09-14T03:43:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:1715: rmattr oid 32 current snap is 167 2024-09-14T03:43:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:1715: done (5 left) 2024-09-14T03:43:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:1716: setattr oid 14 current snap is 167 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1717: read oid 15 snap 140 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1717: expect deleted 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1718: setattr oid 5 current snap is 167 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1714: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1714: done (7 left) 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1719: read oid 10 snap 140 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1719: expect (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1720: rollback oid 39 current snap is 167 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 140 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1721: rollback oid 40 current snap is 167 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 40 to 140 2024-09-14T03:43:27.802 INFO:tasks.rados.rados.0.smithi110.stdout:1722: read oid 12 snap -1 2024-09-14T03:43:27.803 INFO:tasks.rados.rados.0.smithi110.stdout:1722: expect (ObjNum 539 snap 166 seq_num 539) 2024-09-14T03:43:27.803 INFO:tasks.rados.rados.0.smithi110.stdout:1723: read oid 45 snap -1 2024-09-14T03:43:27.803 INFO:tasks.rados.rados.0.smithi110.stdout:1723: expect (ObjNum 519 snap 163 seq_num 519) 2024-09-14T03:43:27.803 INFO:tasks.rados.rados.0.smithi110.stdout:1724: setattr oid 28 current snap is 167 2024-09-14T03:43:27.803 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 167 2024-09-14T03:43:27.803 INFO:tasks.rados.rados.0.smithi110.stdout:1725: seq_num 551 ranges {0=434176,434176=647168,1081344=450560,1531904=507904} 2024-09-14T03:43:27.805 INFO:tasks.rados.rados.0.smithi110.stdout:1725: writing smithi11075562-17 from 0 to 434176 tid 1 2024-09-14T03:43:27.810 INFO:tasks.rados.rados.0.smithi110.stdout:1725: writing smithi11075562-17 from 434176 to 1081344 tid 2 2024-09-14T03:43:27.813 INFO:tasks.rados.rados.0.smithi110.stdout:1725: writing smithi11075562-17 from 1081344 to 1531904 tid 3 2024-09-14T03:43:27.816 INFO:tasks.rados.rados.0.smithi110.stdout:1725: writing smithi11075562-17 from 1531904 to 2039808 tid 4 2024-09-14T03:43:27.816 INFO:tasks.rados.rados.0.smithi110.stdout:1726: snap_create 2024-09-14T03:43:27.816 INFO:tasks.rados.rados.0.smithi110.stdout:1708: expect (ObjNum 531 snap 165 seq_num 531) 2024-09-14T03:43:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:27 smithi110 ceph-mon[86423]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:43:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:27 smithi110 ceph-mon[86423]: osdmap e410: 8 total, 7 up, 8 in 2024-09-14T03:43:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:27 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:27.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:27 smithi110 ceph-mon[88262]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:43:27.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:27 smithi110 ceph-mon[88262]: osdmap e410: 8 total, 7 up, 8 in 2024-09-14T03:43:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:27 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:27.947 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:27 smithi110 podman[106508]: 2024-09-14 03:43:27.652015874 +0000 UTC m=+1.884158748 container cleanup efe1d43f93e1e5defd740e373e2fa178fcd956930044cc2fd927058dffec3b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-eb16b97, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:43:27.947 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:27 smithi110 bash[106508]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3 2024-09-14T03:43:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:27 smithi195 ceph-mon[82388]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:43:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:27 smithi195 ceph-mon[82388]: osdmap e410: 8 total, 7 up, 8 in 2024-09-14T03:43:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:27 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:28.406 INFO:tasks.rados.rados.0.smithi110.stdout:1711: finishing write tid 1 to smithi11075562-33 2024-09-14T03:43:28.406 INFO:tasks.rados.rados.0.smithi110.stdout:1711: finishing write tid 2 to smithi11075562-33 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1711: finishing write tid 3 to smithi11075562-33 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1711: finishing write tid 4 to smithi11075562-33 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1711: finishing write tid 5 to smithi11075562-33 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1016 (ObjNum 500 snap 156 seq_num 500) dirty exists 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1712: finishing rollback tid 0 to smithi11075562-6 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1712: finishing rollback tid 1 to smithi11075562-6 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1711: finishing write tid 6 to smithi11075562-33 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1033 (ObjNum 550 snap 167 seq_num 550) dirty exists 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1711: left oid 33 (ObjNum 550 snap 167 seq_num 550) 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1720: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:1712: finishing rollback tid 2 to smithi11075562-6 2024-09-14T03:43:28.407 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1003 (ObjNum 521 snap 163 seq_num 521) dirty exists 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:1720: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1018 (ObjNum 389 snap 129 seq_num 389) dirty exists 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:1721: finishing rollback tid 0 to smithi11075562-40 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:1721: finishing rollback tid 1 to smithi11075562-40 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:1721: finishing rollback tid 2 to smithi11075562-40 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1007 (ObjNum 411 snap 134 seq_num 411) dirty exists 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:1725: finishing write tid 1 to smithi11075562-17 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 693 (ObjNum 33 snap 0 seq_num 66233712) dirty exists 2024-09-14T03:43:28.408 INFO:tasks.rados.rados.0.smithi110.stdout:1713: expect (ObjNum 418 snap 136 seq_num 418) 2024-09-14T03:43:28.444 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:27 smithi110 podman[106533]: 2024-09-14 03:43:27.970183374 +0000 UTC m=+0.682041415 container remove efe1d43f93e1e5defd740e373e2fa178fcd956930044cc2fd927058dffec3b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-14T03:43:28.463 INFO:tasks.rados.rados.0.smithi110.stdout:1725: finishing write tid 2 to smithi11075562-17 2024-09-14T03:43:28.463 INFO:tasks.rados.rados.0.smithi110.stdout:1725: finishing write tid 3 to smithi11075562-17 2024-09-14T03:43:28.463 INFO:tasks.rados.rados.0.smithi110.stdout:1725: finishing write tid 4 to smithi11075562-17 2024-09-14T03:43:28.463 INFO:tasks.rados.rados.0.smithi110.stdout:1725: finishing write tid 5 to smithi11075562-17 2024-09-14T03:43:28.464 INFO:tasks.rados.rados.0.smithi110.stdout:1725: finishing write tid 6 to smithi11075562-17 2024-09-14T03:43:28.464 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 914 (ObjNum 551 snap 167 seq_num 551) dirty exists 2024-09-14T03:43:28.464 INFO:tasks.rados.rados.0.smithi110.stdout:1725: left oid 17 (ObjNum 551 snap 167 seq_num 551) 2024-09-14T03:43:28.464 INFO:tasks.rados.rados.0.smithi110.stdout:1719: expect (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:43:28.696 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:28 smithi110 podman[106620]: 2024-09-14 03:43:28.346576663 +0000 UTC m=+0.019204572 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:43:28.696 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:28 smithi110 podman[106620]: 2024-09-14 03:43:28.454559273 +0000 UTC m=+0.127187182 container create 716a8de0d3f4075b2a08938901414d62667c36af7677e140cead93fa1a8efc3f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-a278231) 2024-09-14T03:43:28.852 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1304 (ObjNum 542 snap 166 seq_num 542) dirty exists 2024-09-14T03:43:28.852 INFO:tasks.rados.rados.0.smithi110.stdout:1722: expect (ObjNum 539 snap 166 seq_num 539) 2024-09-14T03:43:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:28 smithi110 ceph-mon[88262]: osdmap e411: 8 total, 7 up, 8 in 2024-09-14T03:43:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:28 smithi110 ceph-mon[88262]: pgmap v243: 121 pgs: 24 peering, 6 stale+active+clean, 91 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.2 MiB/s wr, 9 op/s 2024-09-14T03:43:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:28 smithi110 ceph-mon[88262]: osdmap e412: 8 total, 7 up, 8 in 2024-09-14T03:43:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:28 smithi110 ceph-mon[86423]: osdmap e411: 8 total, 7 up, 8 in 2024-09-14T03:43:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:28 smithi110 ceph-mon[86423]: pgmap v243: 121 pgs: 24 peering, 6 stale+active+clean, 91 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.2 MiB/s wr, 9 op/s 2024-09-14T03:43:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:28 smithi110 ceph-mon[86423]: osdmap e412: 8 total, 7 up, 8 in 2024-09-14T03:43:29.197 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:28 smithi110 podman[106620]: 2024-09-14 03:43:28.698963393 +0000 UTC m=+0.371591311 container init 716a8de0d3f4075b2a08938901414d62667c36af7677e140cead93fa1a8efc3f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=main-a278231, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-14T03:43:29.197 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:28 smithi110 podman[106620]: 2024-09-14 03:43:28.703571067 +0000 UTC m=+0.376198978 container start 716a8de0d3f4075b2a08938901414d62667c36af7677e140cead93fa1a8efc3f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-14T03:43:29.197 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:28 smithi110 podman[106620]: 2024-09-14 03:43:28.798965341 +0000 UTC m=+0.471593251 container attach 716a8de0d3f4075b2a08938901414d62667c36af7677e140cead93fa1a8efc3f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-14T03:43:29.197 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:28 smithi110 podman[106620]: 2024-09-14 03:43:28.902652782 +0000 UTC m=+0.575280699 container died 716a8de0d3f4075b2a08938901414d62667c36af7677e140cead93fa1a8efc3f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:43:29.201 INFO:tasks.rados.rados.0.smithi110.stdout:1723: expect (ObjNum 519 snap 163 seq_num 519) 2024-09-14T03:43:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:28 smithi195 ceph-mon[82388]: osdmap e411: 8 total, 7 up, 8 in 2024-09-14T03:43:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:28 smithi195 ceph-mon[82388]: pgmap v243: 121 pgs: 24 peering, 6 stale+active+clean, 91 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.2 MiB/s wr, 9 op/s 2024-09-14T03:43:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:28 smithi195 ceph-mon[82388]: osdmap e412: 8 total, 7 up, 8 in 2024-09-14T03:43:29.399 INFO:tasks.rados.rados.0.smithi110.stdout:1708: done (14 left) 2024-09-14T03:43:29.399 INFO:tasks.rados.rados.0.smithi110.stdout:1711: done (13 left) 2024-09-14T03:43:29.399 INFO:tasks.rados.rados.0.smithi110.stdout:1712: done (12 left) 2024-09-14T03:43:29.399 INFO:tasks.rados.rados.0.smithi110.stdout:1713: done (11 left) 2024-09-14T03:43:29.399 INFO:tasks.rados.rados.0.smithi110.stdout:1716: done (10 left) 2024-09-14T03:43:29.399 INFO:tasks.rados.rados.0.smithi110.stdout:1717: done (9 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1718: done (8 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1719: done (7 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1720: done (6 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1721: done (5 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1722: done (4 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1723: done (3 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1724: done (2 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1725: done (1 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1726: done (0 left) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1727: read oid 27 snap -1 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1727: expect (ObjNum 464 snap 150 seq_num 464) 2024-09-14T03:43:29.400 INFO:tasks.rados.rados.0.smithi110.stdout:1728: delete oid 31 current snap is 168 2024-09-14T03:43:29.402 INFO:tasks.rados.rados.0.smithi110.stdout:1728: done (1 left) 2024-09-14T03:43:29.402 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 168 2024-09-14T03:43:29.402 INFO:tasks.rados.rados.0.smithi110.stdout:1729: seq_num 553 ranges {8151040=466944,8617984=417792,9035776=417792,9453568=516096,9969664=253952} 2024-09-14T03:43:29.448 INFO:tasks.rados.rados.0.smithi110.stdout:1729: writing smithi11075562-44 from 8151040 to 8617984 tid 1 2024-09-14T03:43:29.451 INFO:tasks.rados.rados.0.smithi110.stdout:1729: writing smithi11075562-44 from 8617984 to 9035776 tid 2 2024-09-14T03:43:29.453 INFO:tasks.rados.rados.0.smithi110.stdout:1729: writing smithi11075562-44 from 9035776 to 9453568 tid 3 2024-09-14T03:43:29.456 INFO:tasks.rados.rados.0.smithi110.stdout:1729: writing smithi11075562-44 from 9453568 to 9969664 tid 4 2024-09-14T03:43:29.458 INFO:tasks.rados.rados.0.smithi110.stdout:1729: writing smithi11075562-44 from 9969664 to 10223616 tid 5 2024-09-14T03:43:29.458 INFO:tasks.rados.rados.0.smithi110.stdout:1730: read oid 2 snap -1 2024-09-14T03:43:29.458 INFO:tasks.rados.rados.0.smithi110.stdout:1730: expect (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:43:29.458 INFO:tasks.rados.rados.0.smithi110.stdout:1731: copy_from oid 49 from oid 16 current snap is 168 2024-09-14T03:43:29.458 INFO:tasks.rados.rados.0.smithi110.stdout:1727: expect (ObjNum 464 snap 150 seq_num 464) 2024-09-14T03:43:29.746 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:29 smithi110 podman[106643]: 2024-09-14 03:43:29.452789654 +0000 UTC m=+0.540785101 container remove 716a8de0d3f4075b2a08938901414d62667c36af7677e140cead93fa1a8efc3f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-14T03:43:29.746 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:29 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-14T03:43:29.746 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:29 smithi110 systemd[1]: Stopped Ceph osd.3 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:43:29.746 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:29 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.3.service: Consumed 10.684s CPU time. 2024-09-14T03:43:29.746 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:29 smithi110 systemd[1]: Starting Ceph osd.3 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:43:30.020 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:29 smithi110 ceph-mon[88262]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-14T03:43:30.021 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:29 smithi110 podman[106738]: 2024-09-14 03:43:29.802670081 +0000 UTC m=+0.018716611 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:43:30.021 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:29 smithi110 podman[106738]: 2024-09-14 03:43:29.912186669 +0000 UTC m=+0.128233200 container create d66f1396236b713caf169e27c85a5e3937b2ca679e876c27b679b8d43a09cb8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-14T03:43:30.021 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:29 smithi110 ceph-mon[86423]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-14T03:43:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:1729: finishing write tid 1 to smithi11075562-44 2024-09-14T03:43:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:1727: done (3 left) 2024-09-14T03:43:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:1732: setattr oid 23 current snap is 168 2024-09-14T03:43:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:1729: finishing write tid 2 to smithi11075562-44 2024-09-14T03:43:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:1729: finishing write tid 3 to smithi11075562-44 2024-09-14T03:43:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:1729: finishing write tid 4 to smithi11075562-44 2024-09-14T03:43:30.035 INFO:tasks.rados.rados.0.smithi110.stdout:1729: finishing write tid 5 to smithi11075562-44 2024-09-14T03:43:30.036 INFO:tasks.rados.rados.0.smithi110.stdout:1729: finishing write tid 6 to smithi11075562-44 2024-09-14T03:43:30.036 INFO:tasks.rados.rados.0.smithi110.stdout:1729: finishing write tid 7 to smithi11075562-44 2024-09-14T03:43:30.036 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 399 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:43:30.036 INFO:tasks.rados.rados.0.smithi110.stdout:1729: left oid 44 (ObjNum 553 snap 168 seq_num 553) 2024-09-14T03:43:30.036 INFO:tasks.rados.rados.0.smithi110.stdout:1730: expect (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:43:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:29 smithi195 ceph-mon[82388]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-14T03:43:30.272 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 podman[106738]: 2024-09-14 03:43:30.139694627 +0000 UTC m=+0.355741165 container init d66f1396236b713caf169e27c85a5e3937b2ca679e876c27b679b8d43a09cb8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-a278231, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-14T03:43:30.273 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 podman[106738]: 2024-09-14 03:43:30.14299352 +0000 UTC m=+0.359040057 container start d66f1396236b713caf169e27c85a5e3937b2ca679e876c27b679b8d43a09cb8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:43:30.273 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 podman[106738]: 2024-09-14 03:43:30.223095508 +0000 UTC m=+0.439142036 container attach d66f1396236b713caf169e27c85a5e3937b2ca679e876c27b679b8d43a09cb8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-14T03:43:30.595 INFO:tasks.rados.rados.0.smithi110.stdout:1731: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:43:30.595 INFO:tasks.rados.rados.0.smithi110.stdout:1731: finishing copy_from to smithi11075562-49 2024-09-14T03:43:30.595 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 981 (ObjNum 540 snap 166 seq_num 540) dirty exists 2024-09-14T03:43:30.596 INFO:tasks.rados.rados.0.smithi110.stdout:1729: done (3 left) 2024-09-14T03:43:30.596 INFO:tasks.rados.rados.0.smithi110.stdout:1730: done (2 left) 2024-09-14T03:43:30.596 INFO:tasks.rados.rados.0.smithi110.stdout:1731: done (1 left) 2024-09-14T03:43:30.596 INFO:tasks.rados.rados.0.smithi110.stdout:1733: delete oid 3 current snap is 168 2024-09-14T03:43:30.597 INFO:tasks.rados.rados.0.smithi110.stdout:1733: done (1 left) 2024-09-14T03:43:30.597 INFO:tasks.rados.rados.0.smithi110.stdout:1734: read oid 43 snap 166 2024-09-14T03:43:30.598 INFO:tasks.rados.rados.0.smithi110.stdout:1734: expect (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:43:30.598 INFO:tasks.rados.rados.0.smithi110.stdout:1735: read oid 25 snap -1 2024-09-14T03:43:30.598 INFO:tasks.rados.rados.0.smithi110.stdout:1735: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:43:30.598 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 168 2024-09-14T03:43:30.598 INFO:tasks.rados.rados.0.smithi110.stdout:1736: seq_num 555 ranges {0=499712,499712=638976,1138688=425984,1564672=540672} 2024-09-14T03:43:30.601 INFO:tasks.rados.rados.0.smithi110.stdout:1736: writing smithi11075562-42 from 0 to 499712 tid 1 2024-09-14T03:43:30.604 INFO:tasks.rados.rados.0.smithi110.stdout:1736: writing smithi11075562-42 from 499712 to 1138688 tid 2 2024-09-14T03:43:30.607 INFO:tasks.rados.rados.0.smithi110.stdout:1736: writing smithi11075562-42 from 1138688 to 1564672 tid 3 2024-09-14T03:43:30.610 INFO:tasks.rados.rados.0.smithi110.stdout:1736: writing smithi11075562-42 from 1564672 to 2105344 tid 4 2024-09-14T03:43:30.610 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 168 2024-09-14T03:43:30.610 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 907 (ObjNum 541 snap 166 seq_num 541) dirty exists 2024-09-14T03:43:30.610 INFO:tasks.rados.rados.0.smithi110.stdout:1734: expect (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:43:30.696 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:43:30.696 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 bash[106738]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:43:30.696 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:43:30.696 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 bash[106738]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:43:30.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:30.764 INFO:tasks.rados.rados.0.smithi110.stdout:1736: finishing write tid 1 to smithi11075562-42 2024-09-14T03:43:30.764 INFO:tasks.rados.rados.0.smithi110.stdout:1736: finishing write tid 2 to smithi11075562-42 2024-09-14T03:43:30.764 INFO:tasks.rados.rados.0.smithi110.stdout:1737: seq_num 556 ranges {9453568=516096,9969664=524288,10493952=638976,11132928=442368} 2024-09-14T03:43:30.816 INFO:tasks.rados.rados.0.smithi110.stdout:1737: writing smithi11075562-22 from 9453568 to 9969664 tid 1 2024-09-14T03:43:30.820 INFO:tasks.rados.rados.0.smithi110.stdout:1737: writing smithi11075562-22 from 9969664 to 10493952 tid 2 2024-09-14T03:43:30.824 INFO:tasks.rados.rados.0.smithi110.stdout:1737: writing smithi11075562-22 from 10493952 to 11132928 tid 3 2024-09-14T03:43:30.827 INFO:tasks.rados.rados.0.smithi110.stdout:1737: writing smithi11075562-22 from 11132928 to 11575296 tid 4 2024-09-14T03:43:30.827 INFO:tasks.rados.rados.0.smithi110.stdout:1732: done (4 left) 2024-09-14T03:43:30.827 INFO:tasks.rados.rados.0.smithi110.stdout:1734: done (3 left) 2024-09-14T03:43:30.827 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 168 2024-09-14T03:43:30.827 INFO:tasks.rados.rados.0.smithi110.stdout:1736: finishing write tid 3 to smithi11075562-42 2024-09-14T03:43:30.827 INFO:tasks.rados.rados.0.smithi110.stdout:1736: finishing write tid 4 to smithi11075562-42 2024-09-14T03:43:30.827 INFO:tasks.rados.rados.0.smithi110.stdout:1736: finishing write tid 5 to smithi11075562-42 2024-09-14T03:43:30.828 INFO:tasks.rados.rados.0.smithi110.stdout:1736: finishing write tid 6 to smithi11075562-42 2024-09-14T03:43:30.828 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1101 (ObjNum 555 snap 168 seq_num 555) dirty exists 2024-09-14T03:43:30.828 INFO:tasks.rados.rados.0.smithi110.stdout:1736: left oid 42 (ObjNum 555 snap 168 seq_num 555) 2024-09-14T03:43:30.828 INFO:tasks.rados.rados.0.smithi110.stdout:1735: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:43:31.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:30 smithi110 ceph-mon[88262]: pgmap v245: 121 pgs: 1 active+clean+snaptrim, 1 activating+undersized, 7 active+undersized, 24 peering, 1 stale+active+clean, 3 active+undersized+degraded, 84 active+clean; 303 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1018 B/s rd, 3.8 MiB/s wr, 11 op/s; 14/393 objects degraded (3.562%) 2024-09-14T03:43:31.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:30 smithi110 ceph-mon[86423]: pgmap v245: 121 pgs: 1 active+clean+snaptrim, 1 activating+undersized, 7 active+undersized, 24 peering, 1 stale+active+clean, 3 active+undersized+degraded, 84 active+clean; 303 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1018 B/s rd, 3.8 MiB/s wr, 11 op/s; 14/393 objects degraded (3.562%) 2024-09-14T03:43:31.041 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:43:31.042 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 bash[106738]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:43:31.042 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T03:43:31.042 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:30 smithi110 bash[106738]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T03:43:31.142 INFO:tasks.rados.rados.0.smithi110.stdout:1737: finishing write tid 1 to smithi11075562-22 2024-09-14T03:43:31.142 INFO:tasks.rados.rados.0.smithi110.stdout:1737: finishing write tid 2 to smithi11075562-22 2024-09-14T03:43:31.142 INFO:tasks.rados.rados.0.smithi110.stdout:1737: finishing write tid 3 to smithi11075562-22 2024-09-14T03:43:31.142 INFO:tasks.rados.rados.0.smithi110.stdout:1737: finishing write tid 4 to smithi11075562-22 2024-09-14T03:43:31.142 INFO:tasks.rados.rados.0.smithi110.stdout:1737: finishing write tid 5 to smithi11075562-22 2024-09-14T03:43:31.142 INFO:tasks.rados.rados.0.smithi110.stdout:1737: finishing write tid 6 to smithi11075562-22 2024-09-14T03:43:31.143 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1101 (ObjNum 556 snap 168 seq_num 556) dirty exists 2024-09-14T03:43:31.143 INFO:tasks.rados.rados.0.smithi110.stdout:1737: left oid 22 (ObjNum 556 snap 168 seq_num 556) 2024-09-14T03:43:31.143 INFO:tasks.rados.rados.0.smithi110.stdout:1738: seq_num 557 ranges {8699904=532480,9232384=802816,10035200=794624,10829824=8192} 2024-09-14T03:43:31.190 INFO:tasks.rados.rados.0.smithi110.stdout:1738: writing smithi11075562-14 from 8699904 to 9232384 tid 1 2024-09-14T03:43:31.194 INFO:tasks.rados.rados.0.smithi110.stdout:1738: writing smithi11075562-14 from 9232384 to 10035200 tid 2 2024-09-14T03:43:31.199 INFO:tasks.rados.rados.0.smithi110.stdout:1738: writing smithi11075562-14 from 10035200 to 10829824 tid 3 2024-09-14T03:43:31.199 INFO:tasks.rados.rados.0.smithi110.stdout:1738: writing smithi11075562-14 from 10829824 to 10838016 tid 4 2024-09-14T03:43:31.200 INFO:tasks.rados.rados.0.smithi110.stdout:1735: done (3 left) 2024-09-14T03:43:31.200 INFO:tasks.rados.rados.0.smithi110.stdout:1736: done (2 left) 2024-09-14T03:43:31.200 INFO:tasks.rados.rados.0.smithi110.stdout:1737: done (1 left) 2024-09-14T03:43:31.200 INFO:tasks.rados.rados.0.smithi110.stdout:1739: snap_remove snap 167 2024-09-14T03:43:31.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:30 smithi195 ceph-mon[82388]: pgmap v245: 121 pgs: 1 active+clean+snaptrim, 1 activating+undersized, 7 active+undersized, 24 peering, 1 stale+active+clean, 3 active+undersized+degraded, 84 active+clean; 303 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1018 B/s rd, 3.8 MiB/s wr, 11 op/s; 14/393 objects degraded (3.562%) 2024-09-14T03:43:31.412 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T03:43:31.412 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 bash[106738]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T03:43:31.412 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T03:43:31.412 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 bash[106738]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T03:43:31.412 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-14T03:43:31.413 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 bash[106738]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-14T03:43:31.413 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:43:31.413 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 bash[106738]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:43:31.413 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate[106753]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-14T03:43:31.413 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 bash[106738]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-14T03:43:31.413 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 podman[106738]: 2024-09-14 03:43:31.068355446 +0000 UTC m=+1.284401976 container died d66f1396236b713caf169e27c85a5e3937b2ca679e876c27b679b8d43a09cb8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-14T03:43:31.686 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:43:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:43:31] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-14T03:43:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1739: done (1 left) 2024-09-14T03:43:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1740: snap_create 2024-09-14T03:43:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1738: finishing write tid 1 to smithi11075562-14 2024-09-14T03:43:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1738: finishing write tid 2 to smithi11075562-14 2024-09-14T03:43:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1738: finishing write tid 3 to smithi11075562-14 2024-09-14T03:43:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1738: finishing write tid 4 to smithi11075562-14 2024-09-14T03:43:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1738: finishing write tid 5 to smithi11075562-14 2024-09-14T03:43:31.758 INFO:tasks.rados.rados.0.smithi110.stdout:1738: finishing write tid 6 to smithi11075562-14 2024-09-14T03:43:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1024 (ObjNum 557 snap 168 seq_num 557) dirty exists 2024-09-14T03:43:31.759 INFO:tasks.rados.rados.0.smithi110.stdout:1738: left oid 14 (ObjNum 557 snap 168 seq_num 557) 2024-09-14T03:43:31.938 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:31 smithi110 ceph-mon[86423]: Health check failed: Degraded data redundancy: 14/393 objects degraded (3.562%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:31.938 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:31 smithi110 ceph-mon[88262]: Health check failed: Degraded data redundancy: 14/393 objects degraded (3.562%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:31.938 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 podman[106894]: 2024-09-14 03:43:31.686100963 +0000 UTC m=+0.607972274 container remove d66f1396236b713caf169e27c85a5e3937b2ca679e876c27b679b8d43a09cb8a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-a278231) 2024-09-14T03:43:31.938 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 podman[106950]: 2024-09-14 03:43:31.792395889 +0000 UTC m=+0.015839787 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:43:32.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:31 smithi110 podman[106950]: 2024-09-14 03:43:31.937370613 +0000 UTC m=+0.160814505 container create 2283ebcb5f1004d507c60ac6e1c762b6c8ecf9507da384880ecd68b6a58fb17e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-14T03:43:32.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:32 smithi110 podman[106950]: 2024-09-14 03:43:32.156356953 +0000 UTC m=+0.379800844 container init 2283ebcb5f1004d507c60ac6e1c762b6c8ecf9507da384880ecd68b6a58fb17e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-14T03:43:32.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:32 smithi110 podman[106950]: 2024-09-14 03:43:32.159593996 +0000 UTC m=+0.383037890 container start 2283ebcb5f1004d507c60ac6e1c762b6c8ecf9507da384880ecd68b6a58fb17e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:43:32.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-osd[106970]: -- 172.21.15.110:0/1404227148 <== mon.0 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5630bcff1a40 con 0x5630bcfcfc00 2024-09-14T03:43:32.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:31 smithi195 ceph-mon[82388]: Health check failed: Degraded data redundancy: 14/393 objects degraded (3.562%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:32.696 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:32 smithi110 bash[106950]: 2283ebcb5f1004d507c60ac6e1c762b6c8ecf9507da384880ecd68b6a58fb17e 2024-09-14T03:43:32.696 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:32 smithi110 systemd[1]: Started Ceph osd.3 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:43:32.757 INFO:tasks.rados.rados.0.smithi110.stdout:1738: done (1 left) 2024-09-14T03:43:32.757 INFO:tasks.rados.rados.0.smithi110.stdout:1740: done (0 left) 2024-09-14T03:43:32.757 INFO:tasks.rados.rados.0.smithi110.stdout:1741: read oid 34 snap -1 2024-09-14T03:43:32.757 INFO:tasks.rados.rados.0.smithi110.stdout:1741: expect deleted 2024-09-14T03:43:32.757 INFO:tasks.rados.rados.0.smithi110.stdout:1742: setattr oid 5 current snap is 169 2024-09-14T03:43:32.758 INFO:tasks.rados.rados.0.smithi110.stdout:1743: snap_create 2024-09-14T03:43:32.765 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1306 (ObjNum 542 snap 166 seq_num 542) dirty exists 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[88262]: pgmap v246: 121 pgs: 1 active+clean+snaptrim, 1 activating+undersized, 10 active+undersized, 24 peering, 5 active+undersized+degraded, 80 active+clean; 290 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.5 MiB/s wr, 9 op/s; 23/366 objects degraded (6.284%) 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[88262]: osdmap e413: 8 total, 7 up, 8 in 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[86423]: pgmap v246: 121 pgs: 1 active+clean+snaptrim, 1 activating+undersized, 10 active+undersized, 24 peering, 5 active+undersized+degraded, 80 active+clean; 290 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.5 MiB/s wr, 9 op/s; 23/366 objects degraded (6.284%) 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[86423]: osdmap e413: 8 total, 7 up, 8 in 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:33.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:33.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:32 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:32 smithi195 ceph-mon[82388]: pgmap v246: 121 pgs: 1 active+clean+snaptrim, 1 activating+undersized, 10 active+undersized, 24 peering, 5 active+undersized+degraded, 80 active+clean; 290 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.5 MiB/s wr, 9 op/s; 23/366 objects degraded (6.284%) 2024-09-14T03:43:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:32 smithi195 ceph-mon[82388]: osdmap e413: 8 total, 7 up, 8 in 2024-09-14T03:43:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:32 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:32 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:32 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:33.701 INFO:tasks.rados.rados.0.smithi110.stdout:1741: done (2 left) 2024-09-14T03:43:33.701 INFO:tasks.rados.rados.0.smithi110.stdout:1742: done (1 left) 2024-09-14T03:43:33.701 INFO:tasks.rados.rados.0.smithi110.stdout:1743: done (0 left) 2024-09-14T03:43:33.701 INFO:tasks.rados.rados.0.smithi110.stdout:1744: delete oid 6 current snap is 170 2024-09-14T03:43:33.703 INFO:tasks.rados.rados.0.smithi110.stdout:1744: done (0 left) 2024-09-14T03:43:33.703 INFO:tasks.rados.rados.0.smithi110.stdout:1745: snap_create 2024-09-14T03:43:34.089 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:33 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[106966]: 2024-09-14T03:43:33.810+0000 7fdb9dad1640 -1 Falling back to public interface 2024-09-14T03:43:34.090 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:33 smithi110 ceph-mon[88262]: osdmap e414: 8 total, 7 up, 8 in 2024-09-14T03:43:34.090 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:33 smithi110 ceph-mon[88262]: osdmap e415: 8 total, 7 up, 8 in 2024-09-14T03:43:34.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:33 smithi110 ceph-mon[86423]: osdmap e414: 8 total, 7 up, 8 in 2024-09-14T03:43:34.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:33 smithi110 ceph-mon[86423]: osdmap e415: 8 total, 7 up, 8 in 2024-09-14T03:43:34.148 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:33 smithi195 ceph-mon[82388]: osdmap e414: 8 total, 7 up, 8 in 2024-09-14T03:43:34.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:33 smithi195 ceph-mon[82388]: osdmap e415: 8 total, 7 up, 8 in 2024-09-14T03:43:34.458 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:34 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:34.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:43:34.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:34 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:34.705 INFO:tasks.rados.rados.0.smithi110.stdout:1745: done (0 left) 2024-09-14T03:43:34.705 INFO:tasks.rados.rados.0.smithi110.stdout:1746: snap_remove snap 140 2024-09-14T03:43:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:34 smithi110 ceph-mon[88262]: pgmap v250: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 316 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.2 MiB/s wr, 27 op/s; 35/375 objects degraded (9.333%) 2024-09-14T03:43:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:34 smithi110 ceph-mon[88262]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-14T03:43:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:34 smithi110 ceph-mon[88262]: osdmap e416: 8 total, 7 up, 8 in 2024-09-14T03:43:34.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:34 smithi110 ceph-mon[86423]: pgmap v250: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 316 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.2 MiB/s wr, 27 op/s; 35/375 objects degraded (9.333%) 2024-09-14T03:43:34.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:34 smithi110 ceph-mon[86423]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-14T03:43:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:34 smithi110 ceph-mon[86423]: osdmap e416: 8 total, 7 up, 8 in 2024-09-14T03:43:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:34 smithi195 ceph-mon[82388]: pgmap v250: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 316 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.2 MiB/s wr, 27 op/s; 35/375 objects degraded (9.333%) 2024-09-14T03:43:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:34 smithi195 ceph-mon[82388]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-14T03:43:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:34 smithi195 ceph-mon[82388]: osdmap e416: 8 total, 7 up, 8 in 2024-09-14T03:43:35.707 INFO:tasks.rados.rados.0.smithi110.stdout:1746: done (0 left) 2024-09-14T03:43:35.707 INFO:tasks.rados.rados.0.smithi110.stdout:1747: delete oid 32 current snap is 171 2024-09-14T03:43:35.719 INFO:tasks.rados.rados.0.smithi110.stdout:1747: done (0 left) 2024-09-14T03:43:35.720 INFO:tasks.rados.rados.0.smithi110.stdout:1748: delete oid 21 current snap is 171 2024-09-14T03:43:35.724 INFO:tasks.rados.rados.0.smithi110.stdout:1748: done (0 left) 2024-09-14T03:43:35.725 INFO:tasks.rados.rados.0.smithi110.stdout:1749: snap_remove snap 169 2024-09-14T03:43:36.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:36.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:35 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:36.712 INFO:tasks.rados.rados.0.smithi110.stdout:1749: done (0 left) 2024-09-14T03:43:36.713 INFO:tasks.rados.rados.0.smithi110.stdout:1750: snap_remove snap 170 2024-09-14T03:43:36.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[86423]: osdmap e417: 8 total, 7 up, 8 in 2024-09-14T03:43:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[86423]: pgmap v253: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 4.5 MiB/s wr, 35 op/s; 36/384 objects degraded (9.375%) 2024-09-14T03:43:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[88262]: osdmap e417: 8 total, 7 up, 8 in 2024-09-14T03:43:36.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[88262]: pgmap v253: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 4.5 MiB/s wr, 35 op/s; 36/384 objects degraded (9.375%) 2024-09-14T03:43:36.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:36 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:36.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:36 smithi195 ceph-mon[82388]: osdmap e417: 8 total, 7 up, 8 in 2024-09-14T03:43:36.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:36 smithi195 ceph-mon[82388]: pgmap v253: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 4.5 MiB/s wr, 35 op/s; 36/384 objects degraded (9.375%) 2024-09-14T03:43:36.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:36 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:36 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:36 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:36.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:36 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:37.630 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:37 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[106966]: 2024-09-14T03:43:37.466+0000 7fdb9dad1640 -1 osd.3 409 log_to_monitors true 2024-09-14T03:43:37.717 INFO:tasks.rados.rados.0.smithi110.stdout:1750: done (0 left) 2024-09-14T03:43:37.717 INFO:tasks.rados.rados.0.smithi110.stdout:1751: copy_from oid 11 from oid 16 current snap is 171 2024-09-14T03:43:37.717 INFO:tasks.rados.rados.0.smithi110.stdout:1752: rollback oid 40 current snap is 171 2024-09-14T03:43:37.717 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 40 to 166 2024-09-14T03:43:37.717 INFO:tasks.rados.rados.0.smithi110.stdout:1753: read oid 17 snap -1 2024-09-14T03:43:37.717 INFO:tasks.rados.rados.0.smithi110.stdout:1753: expect (ObjNum 551 snap 167 seq_num 551) 2024-09-14T03:43:37.718 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 171 2024-09-14T03:43:37.718 INFO:tasks.rados.rados.0.smithi110.stdout:1754: seq_num 561 ranges {0=598016,598016=745472,1343488=778240,2121728=81920} 2024-09-14T03:43:37.721 INFO:tasks.rados.rados.0.smithi110.stdout:1754: writing smithi11075562-3 from 0 to 598016 tid 1 2024-09-14T03:43:37.727 INFO:tasks.rados.rados.0.smithi110.stdout:1754: writing smithi11075562-3 from 598016 to 1343488 tid 2 2024-09-14T03:43:37.733 INFO:tasks.rados.rados.0.smithi110.stdout:1754: writing smithi11075562-3 from 1343488 to 2121728 tid 3 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1754: writing smithi11075562-3 from 2121728 to 2203648 tid 4 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1755: copy_from oid 41 from oid 24 current snap is 171 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1754: finishing write tid 1 to smithi11075562-3 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1752: finishing rollback tid 0 to smithi11075562-40 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1752: finishing rollback tid 1 to smithi11075562-40 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1752: finishing rollback tid 2 to smithi11075562-40 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1013 (ObjNum 530 snap 164 seq_num 530) dirty exists 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1754: finishing write tid 2 to smithi11075562-3 2024-09-14T03:43:37.734 INFO:tasks.rados.rados.0.smithi110.stdout:1753: expect (ObjNum 551 snap 167 seq_num 551) 2024-09-14T03:43:37.796 INFO:tasks.rados.rados.0.smithi110.stdout:1754: finishing write tid 3 to smithi11075562-3 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1754: finishing write tid 4 to smithi11075562-3 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1754: finishing write tid 5 to smithi11075562-3 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1751: finishing copy_from racing read to smithi11075562-11 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1751: finishing copy_from to smithi11075562-11 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 981 (ObjNum 540 snap 166 seq_num 540) dirty exists 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1754: finishing write tid 6 to smithi11075562-3 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 699 (ObjNum 561 snap 171 seq_num 561) dirty exists 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1754: left oid 3 (ObjNum 561 snap 171 seq_num 561) 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1755: finishing copy_from racing read to smithi11075562-41 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1751: done (4 left) 2024-09-14T03:43:37.797 INFO:tasks.rados.rados.0.smithi110.stdout:1752: done (3 left) 2024-09-14T03:43:37.798 INFO:tasks.rados.rados.0.smithi110.stdout:1753: done (2 left) 2024-09-14T03:43:37.798 INFO:tasks.rados.rados.0.smithi110.stdout:1754: done (1 left) 2024-09-14T03:43:37.798 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 171 2024-09-14T03:43:37.798 INFO:tasks.rados.rados.0.smithi110.stdout:1755: finishing copy_from to smithi11075562-41 2024-09-14T03:43:37.798 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 899 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:43:37.798 INFO:tasks.rados.rados.0.smithi110.stdout:1756: seq_num 562 ranges {6406144=622592,7028736=630784,7659520=581632,8241152=393216} 2024-09-14T03:43:37.833 INFO:tasks.rados.rados.0.smithi110.stdout:1756: writing smithi11075562-48 from 6406144 to 7028736 tid 1 2024-09-14T03:43:37.837 INFO:tasks.rados.rados.0.smithi110.stdout:1756: writing smithi11075562-48 from 7028736 to 7659520 tid 2 2024-09-14T03:43:37.840 INFO:tasks.rados.rados.0.smithi110.stdout:1756: writing smithi11075562-48 from 7659520 to 8241152 tid 3 2024-09-14T03:43:37.843 INFO:tasks.rados.rados.0.smithi110.stdout:1756: writing smithi11075562-48 from 8241152 to 8634368 tid 4 2024-09-14T03:43:37.843 INFO:tasks.rados.rados.0.smithi110.stdout:1755: done (1 left) 2024-09-14T03:43:37.843 INFO:tasks.rados.rados.0.smithi110.stdout:1757: copy_from oid 18 from oid 20 current snap is 171 2024-09-14T03:43:37.843 INFO:tasks.rados.rados.0.smithi110.stdout:1756: finishing write tid 1 to smithi11075562-48 2024-09-14T03:43:37.843 INFO:tasks.rados.rados.0.smithi110.stdout:1756: finishing write tid 2 to smithi11075562-48 2024-09-14T03:43:37.843 INFO:tasks.rados.rados.0.smithi110.stdout:1758: delete oid 45 current snap is 171 2024-09-14T03:43:37.843 INFO:tasks.rados.rados.0.smithi110.stdout:1756: finishing write tid 3 to smithi11075562-48 2024-09-14T03:43:37.845 INFO:tasks.rados.rados.0.smithi110.stdout:1758: done (2 left) 2024-09-14T03:43:37.845 INFO:tasks.rados.rados.0.smithi110.stdout:1759: snap_remove snap 166 2024-09-14T03:43:37.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:37 smithi110 ceph-mon[86423]: osdmap e418: 8 total, 7 up, 8 in 2024-09-14T03:43:37.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:37 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 36/384 objects degraded (9.375%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:37.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:37 smithi110 ceph-mon[86423]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T03:43:37.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:37 smithi110 ceph-mon[88262]: osdmap e418: 8 total, 7 up, 8 in 2024-09-14T03:43:37.936 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:37 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 36/384 objects degraded (9.375%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:37.936 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:37 smithi110 ceph-mon[88262]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T03:43:37.936 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:43:37 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[106966]: 2024-09-14T03:43:37.740+0000 7fdb9587b640 -1 osd.3 409 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:43:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:37 smithi195 ceph-mon[82388]: osdmap e418: 8 total, 7 up, 8 in 2024-09-14T03:43:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:37 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 36/384 objects degraded (9.375%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:37 smithi195 ceph-mon[82388]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T03:43:38.719 INFO:tasks.rados.rados.0.smithi110.stdout:1759: done (2 left) 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1760: read oid 1 snap 168 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1760: expect deleted 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1756: finishing write tid 4 to smithi11075562-48 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1756: finishing write tid 5 to smithi11075562-48 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1756: finishing write tid 6 to smithi11075562-48 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 986 (ObjNum 562 snap 171 seq_num 562) dirty exists 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1756: left oid 48 (ObjNum 562 snap 171 seq_num 562) 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1757: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1757: finishing copy_from to smithi11075562-18 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1014 (ObjNum 515 snap 160 seq_num 515) dirty exists 2024-09-14T03:43:38.720 INFO:tasks.rados.rados.0.smithi110.stdout:1756: done (2 left) 2024-09-14T03:43:38.721 INFO:tasks.rados.rados.0.smithi110.stdout:1757: done (1 left) 2024-09-14T03:43:38.721 INFO:tasks.rados.rados.0.smithi110.stdout:1761: rmattr oid 19 current snap is 171 2024-09-14T03:43:38.721 INFO:tasks.rados.rados.0.smithi110.stdout:1761: done (1 left) 2024-09-14T03:43:38.721 INFO:tasks.rados.rados.0.smithi110.stdout:1762: read oid 45 snap -1 2024-09-14T03:43:38.721 INFO:tasks.rados.rados.0.smithi110.stdout:1762: expect deleted 2024-09-14T03:43:38.721 INFO:tasks.rados.rados.0.smithi110.stdout:1763: copy_from oid 13 from oid 33 current snap is 171 2024-09-14T03:43:38.721 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 171 2024-09-14T03:43:38.721 INFO:tasks.rados.rados.0.smithi110.stdout:1764: seq_num 564 ranges {2105344=655360,2760704=442368,3203072=598016,3801088=565248} 2024-09-14T03:43:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:1764: writing smithi11075562-42 from 2105344 to 2760704 tid 1 2024-09-14T03:43:38.740 INFO:tasks.rados.rados.0.smithi110.stdout:1764: writing smithi11075562-42 from 2760704 to 3203072 tid 2 2024-09-14T03:43:38.744 INFO:tasks.rados.rados.0.smithi110.stdout:1764: writing smithi11075562-42 from 3203072 to 3801088 tid 3 2024-09-14T03:43:38.748 INFO:tasks.rados.rados.0.smithi110.stdout:1764: writing smithi11075562-42 from 3801088 to 4366336 tid 4 2024-09-14T03:43:38.749 INFO:tasks.rados.rados.0.smithi110.stdout:1765: setattr oid 9 current snap is 171 2024-09-14T03:43:38.749 INFO:tasks.rados.rados.0.smithi110.stdout:1764: finishing write tid 1 to smithi11075562-42 2024-09-14T03:43:38.751 INFO:tasks.rados.rados.0.smithi110.stdout:1760: done (4 left) 2024-09-14T03:43:38.751 INFO:tasks.rados.rados.0.smithi110.stdout:1762: done (3 left) 2024-09-14T03:43:38.751 INFO:tasks.rados.rados.0.smithi110.stdout:1766: snap_remove snap 168 2024-09-14T03:43:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:38 smithi110 ceph-mon[86423]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:43:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:38 smithi110 ceph-mon[86423]: osdmap e419: 8 total, 7 up, 8 in 2024-09-14T03:43:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:38 smithi110 ceph-mon[86423]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:43:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:38 smithi110 ceph-mon[86423]: pgmap v256: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 276 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 518 KiB/s wr, 14 op/s; 36/342 objects degraded (10.526%) 2024-09-14T03:43:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:38 smithi110 ceph-mon[88262]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:43:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:38 smithi110 ceph-mon[88262]: osdmap e419: 8 total, 7 up, 8 in 2024-09-14T03:43:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:38 smithi110 ceph-mon[88262]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:43:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:38 smithi110 ceph-mon[88262]: pgmap v256: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 276 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 518 KiB/s wr, 14 op/s; 36/342 objects degraded (10.526%) 2024-09-14T03:43:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:38 smithi195 ceph-mon[82388]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:43:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:38 smithi195 ceph-mon[82388]: osdmap e419: 8 total, 7 up, 8 in 2024-09-14T03:43:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:38 smithi195 ceph-mon[82388]: from='osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-14T03:43:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:38 smithi195 ceph-mon[82388]: pgmap v256: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 276 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 518 KiB/s wr, 14 op/s; 36/342 objects degraded (10.526%) 2024-09-14T03:43:39.720 INFO:tasks.rados.rados.0.smithi110.stdout:1766: done (3 left) 2024-09-14T03:43:39.720 INFO:tasks.rados.rados.0.smithi110.stdout:1767: rmattr oid 1 current snap is 171 2024-09-14T03:43:39.720 INFO:tasks.rados.rados.0.smithi110.stdout:1767: done (3 left) 2024-09-14T03:43:39.720 INFO:tasks.rados.rados.0.smithi110.stdout:1768: read oid 25 snap -1 2024-09-14T03:43:39.720 INFO:tasks.rados.rados.0.smithi110.stdout:1768: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:43:39.721 INFO:tasks.rados.rados.0.smithi110.stdout:1769: read oid 19 snap -1 2024-09-14T03:43:39.721 INFO:tasks.rados.rados.0.smithi110.stdout:1769: expect deleted 2024-09-14T03:43:39.721 INFO:tasks.rados.rados.0.smithi110.stdout:1770: setattr oid 46 current snap is 171 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:1771: snap_create 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1034 (ObjNum 543 snap 166 seq_num 543) dirty exists 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:1763: finishing copy_from racing read to smithi11075562-13 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:1764: finishing write tid 2 to smithi11075562-42 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:1764: finishing write tid 3 to smithi11075562-42 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:1763: finishing copy_from to smithi11075562-13 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1033 (ObjNum 550 snap 167 seq_num 550) dirty exists 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:1764: finishing write tid 4 to smithi11075562-42 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:1764: finishing write tid 5 to smithi11075562-42 2024-09-14T03:43:39.724 INFO:tasks.rados.rados.0.smithi110.stdout:1764: finishing write tid 6 to smithi11075562-42 2024-09-14T03:43:39.725 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1123 (ObjNum 564 snap 171 seq_num 564) dirty exists 2024-09-14T03:43:39.725 INFO:tasks.rados.rados.0.smithi110.stdout:1764: left oid 42 (ObjNum 564 snap 171 seq_num 564) 2024-09-14T03:43:39.736 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1035 (ObjNum 526 snap 164 seq_num 526) dirty exists 2024-09-14T03:43:39.769 INFO:tasks.rados.rados.0.smithi110.stdout:1768: expect (ObjNum 420 snap 136 seq_num 420) 2024-09-14T03:43:39.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:43:39.651+0000 7f940e88b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-14T03:43:39.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:43:39.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450] boot 2024-09-14T03:43:39.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T03:43:39.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:43:39.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[86423]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:43:39.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450] boot 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:43:39.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:39 smithi110 ceph-mon[88262]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T03:43:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:43:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: osd.3 [v2:172.21.15.110:6826/570098450,v1:172.21.15.110:6827/570098450] boot 2024-09-14T03:43:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T03:43:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:43:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:43:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:43:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:43:40.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:39 smithi195 ceph-mon[82388]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T03:43:40.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:40.721 INFO:tasks.rados.rados.0.smithi110.stdout:1763: done (6 left) 2024-09-14T03:43:40.722 INFO:tasks.rados.rados.0.smithi110.stdout:1764: done (5 left) 2024-09-14T03:43:40.722 INFO:tasks.rados.rados.0.smithi110.stdout:1765: done (4 left) 2024-09-14T03:43:40.722 INFO:tasks.rados.rados.0.smithi110.stdout:1768: done (3 left) 2024-09-14T03:43:40.722 INFO:tasks.rados.rados.0.smithi110.stdout:1769: done (2 left) 2024-09-14T03:43:40.722 INFO:tasks.rados.rados.0.smithi110.stdout:1770: done (1 left) 2024-09-14T03:43:40.722 INFO:tasks.rados.rados.0.smithi110.stdout:1771: done (0 left) 2024-09-14T03:43:40.722 INFO:tasks.rados.rados.0.smithi110.stdout:1772: snap_create 2024-09-14T03:43:41.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:40 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:43:41.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:40 smithi110 ceph-mon[88262]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-14T03:43:41.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:40 smithi110 ceph-mon[88262]: pgmap v259: 121 pgs: 5 peering, 29 active+undersized, 6 active+undersized+degraded, 81 active+clean; 260 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 538 KiB/s wr, 8 op/s; 34/324 objects degraded (10.494%) 2024-09-14T03:43:41.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:40 smithi110 ceph-mon[88262]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T03:43:41.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:40 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:43:41.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:40 smithi110 ceph-mon[86423]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-14T03:43:41.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:40 smithi110 ceph-mon[86423]: pgmap v259: 121 pgs: 5 peering, 29 active+undersized, 6 active+undersized+degraded, 81 active+clean; 260 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 538 KiB/s wr, 8 op/s; 34/324 objects degraded (10.494%) 2024-09-14T03:43:41.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:40 smithi110 ceph-mon[86423]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T03:43:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:40 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:43:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:40 smithi195 ceph-mon[82388]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-14T03:43:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:40 smithi195 ceph-mon[82388]: pgmap v259: 121 pgs: 5 peering, 29 active+undersized, 6 active+undersized+degraded, 81 active+clean; 260 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 538 KiB/s wr, 8 op/s; 34/324 objects degraded (10.494%) 2024-09-14T03:43:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:40 smithi195 ceph-mon[82388]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T03:43:41.723 INFO:tasks.rados.rados.0.smithi110.stdout:1772: done (0 left) 2024-09-14T03:43:41.724 INFO:tasks.rados.rados.0.smithi110.stdout:1773: snap_remove snap 172 2024-09-14T03:43:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:43:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:43:41] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-14T03:43:42.730 INFO:tasks.rados.rados.0.smithi110.stdout:1773: done (0 left) 2024-09-14T03:43:42.730 INFO:tasks.rados.rados.0.smithi110.stdout:1774: rollback oid 7 current snap is 173 2024-09-14T03:43:42.730 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 7 to 171 2024-09-14T03:43:42.730 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 173 2024-09-14T03:43:42.730 INFO:tasks.rados.rados.0.smithi110.stdout:1775: seq_num 565 ranges {0=671744,671744=729088,1400832=753664,2154496=122880} 2024-09-14T03:43:42.734 INFO:tasks.rados.rados.0.smithi110.stdout:1775: writing smithi11075562-31 from 0 to 671744 tid 1 2024-09-14T03:43:42.739 INFO:tasks.rados.rados.0.smithi110.stdout:1775: writing smithi11075562-31 from 671744 to 1400832 tid 2 2024-09-14T03:43:42.745 INFO:tasks.rados.rados.0.smithi110.stdout:1775: writing smithi11075562-31 from 1400832 to 2154496 tid 3 2024-09-14T03:43:42.746 INFO:tasks.rados.rados.0.smithi110.stdout:1775: writing smithi11075562-31 from 2154496 to 2277376 tid 4 2024-09-14T03:43:42.746 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 173 2024-09-14T03:43:42.746 INFO:tasks.rados.rados.0.smithi110.stdout:1776: seq_num 566 ranges {0=688128,688128=614400,1302528=565248,1867776=425984} 2024-09-14T03:43:42.750 INFO:tasks.rados.rados.0.smithi110.stdout:1776: writing smithi11075562-50 from 0 to 688128 tid 1 2024-09-14T03:43:42.754 INFO:tasks.rados.rados.0.smithi110.stdout:1776: writing smithi11075562-50 from 688128 to 1302528 tid 2 2024-09-14T03:43:42.757 INFO:tasks.rados.rados.0.smithi110.stdout:1776: writing smithi11075562-50 from 1302528 to 1867776 tid 3 2024-09-14T03:43:42.761 INFO:tasks.rados.rados.0.smithi110.stdout:1776: writing smithi11075562-50 from 1867776 to 2293760 tid 4 2024-09-14T03:43:42.761 INFO:tasks.rados.rados.0.smithi110.stdout:1777: read oid 9 snap -1 2024-09-14T03:43:42.761 INFO:tasks.rados.rados.0.smithi110.stdout:1777: expect (ObjNum 543 snap 166 seq_num 543) 2024-09-14T03:43:42.761 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 173 2024-09-14T03:43:42.761 INFO:tasks.rados.rados.0.smithi110.stdout:1774: finishing rollback tid 0 to smithi11075562-7 2024-09-14T03:43:42.761 INFO:tasks.rados.rados.0.smithi110.stdout:1778: seq_num 567 ranges {4505600=704512,5210112=499712,5709824=770048,6479872=335872} 2024-09-14T03:43:42.791 INFO:tasks.rados.rados.0.smithi110.stdout:1778: writing smithi11075562-49 from 4505600 to 5210112 tid 1 2024-09-14T03:43:42.794 INFO:tasks.rados.rados.0.smithi110.stdout:1778: writing smithi11075562-49 from 5210112 to 5709824 tid 2 2024-09-14T03:43:42.798 INFO:tasks.rados.rados.0.smithi110.stdout:1778: writing smithi11075562-49 from 5709824 to 6479872 tid 3 2024-09-14T03:43:42.801 INFO:tasks.rados.rados.0.smithi110.stdout:1778: writing smithi11075562-49 from 6479872 to 6815744 tid 4 2024-09-14T03:43:42.801 INFO:tasks.rados.rados.0.smithi110.stdout:1779: read oid 26 snap -1 2024-09-14T03:43:42.801 INFO:tasks.rados.rados.0.smithi110.stdout:1779: expect (ObjNum 520 snap 163 seq_num 520) 2024-09-14T03:43:42.801 INFO:tasks.rados.rados.0.smithi110.stdout:1780: setattr oid 36 current snap is 173 2024-09-14T03:43:42.801 INFO:tasks.rados.rados.0.smithi110.stdout:1774: finishing rollback tid 1 to smithi11075562-7 2024-09-14T03:43:42.801 INFO:tasks.rados.rados.0.smithi110.stdout:1774: finishing rollback tid 2 to smithi11075562-7 2024-09-14T03:43:42.801 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 581 (ObjNum 470 snap 150 seq_num 470) dirty exists 2024-09-14T03:43:42.801 INFO:tasks.rados.rados.0.smithi110.stdout:1775: finishing write tid 1 to smithi11075562-31 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1775: finishing write tid 2 to smithi11075562-31 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1775: finishing write tid 3 to smithi11075562-31 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1775: finishing write tid 4 to smithi11075562-31 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1775: finishing write tid 5 to smithi11075562-31 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1775: finishing write tid 6 to smithi11075562-31 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 912 (ObjNum 565 snap 173 seq_num 565) dirty exists 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1775: left oid 31 (ObjNum 565 snap 173 seq_num 565) 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1776: finishing write tid 1 to smithi11075562-50 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1776: finishing write tid 2 to smithi11075562-50 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1776: finishing write tid 3 to smithi11075562-50 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1776: finishing write tid 4 to smithi11075562-50 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1776: finishing write tid 5 to smithi11075562-50 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:1776: finishing write tid 6 to smithi11075562-50 2024-09-14T03:43:42.802 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 935 (ObjNum 566 snap 173 seq_num 566) dirty exists 2024-09-14T03:43:42.803 INFO:tasks.rados.rados.0.smithi110.stdout:1776: left oid 50 (ObjNum 566 snap 173 seq_num 566) 2024-09-14T03:43:42.803 INFO:tasks.rados.rados.0.smithi110.stdout:1777: expect (ObjNum 543 snap 166 seq_num 543) 2024-09-14T03:43:43.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:42 smithi195 ceph-mon[82388]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T03:43:43.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:42 smithi195 ceph-mon[82388]: pgmap v262: 121 pgs: 1 active+recovering+degraded, 5 peering, 27 active+undersized, 4 active+undersized+degraded, 84 active+clean; 233 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 538 KiB/s wr, 2 op/s; 28/297 objects degraded (9.428%) 2024-09-14T03:43:43.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:43.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:1778: finishing write tid 1 to smithi11075562-49 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:1778: finishing write tid 2 to smithi11075562-49 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:1778: finishing write tid 3 to smithi11075562-49 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:1778: finishing write tid 4 to smithi11075562-49 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:1778: finishing write tid 5 to smithi11075562-49 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:1778: finishing write tid 6 to smithi11075562-49 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1133 (ObjNum 567 snap 173 seq_num 567) dirty exists 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:1778: left oid 49 (ObjNum 567 snap 173 seq_num 567) 2024-09-14T03:43:43.058 INFO:tasks.rados.rados.0.smithi110.stdout:1779: expect (ObjNum 520 snap 163 seq_num 520) 2024-09-14T03:43:43.120 INFO:tasks.rados.rados.0.smithi110.stdout:1774: done (6 left) 2024-09-14T03:43:43.120 INFO:tasks.rados.rados.0.smithi110.stdout:1775: done (5 left) 2024-09-14T03:43:43.120 INFO:tasks.rados.rados.0.smithi110.stdout:1776: done (4 left) 2024-09-14T03:43:43.120 INFO:tasks.rados.rados.0.smithi110.stdout:1777: done (3 left) 2024-09-14T03:43:43.120 INFO:tasks.rados.rados.0.smithi110.stdout:1778: done (2 left) 2024-09-14T03:43:43.120 INFO:tasks.rados.rados.0.smithi110.stdout:1779: done (1 left) 2024-09-14T03:43:43.120 INFO:tasks.rados.rados.0.smithi110.stdout:1781: setattr oid 8 current snap is 173 2024-09-14T03:43:43.121 INFO:tasks.rados.rados.0.smithi110.stdout:1782: copy_from oid 42 from oid 2 current snap is 173 2024-09-14T03:43:43.121 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 173 2024-09-14T03:43:43.121 INFO:tasks.rados.rados.0.smithi110.stdout:1783: seq_num 568 ranges {1122304=720896,1843200=786432,2629632=581632,3211264=237568} 2024-09-14T03:43:43.132 INFO:tasks.rados.rados.0.smithi110.stdout:1783: writing smithi11075562-7 from 1122304 to 1843200 tid 1 2024-09-14T03:43:43.136 INFO:tasks.rados.rados.0.smithi110.stdout:1783: writing smithi11075562-7 from 1843200 to 2629632 tid 2 2024-09-14T03:43:43.140 INFO:tasks.rados.rados.0.smithi110.stdout:1783: writing smithi11075562-7 from 2629632 to 3211264 tid 3 2024-09-14T03:43:43.141 INFO:tasks.rados.rados.0.smithi110.stdout:1783: writing smithi11075562-7 from 3211264 to 3448832 tid 4 2024-09-14T03:43:43.141 INFO:tasks.rados.rados.0.smithi110.stdout:1784: snap_remove snap 171 2024-09-14T03:43:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:42 smithi110 ceph-mon[86423]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T03:43:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:42 smithi110 ceph-mon[86423]: pgmap v262: 121 pgs: 1 active+recovering+degraded, 5 peering, 27 active+undersized, 4 active+undersized+degraded, 84 active+clean; 233 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 538 KiB/s wr, 2 op/s; 28/297 objects degraded (9.428%) 2024-09-14T03:43:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:43.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:42 smithi110 ceph-mon[88262]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T03:43:43.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:42 smithi110 ceph-mon[88262]: pgmap v262: 121 pgs: 1 active+recovering+degraded, 5 peering, 27 active+undersized, 4 active+undersized+degraded, 84 active+clean; 233 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 538 KiB/s wr, 2 op/s; 28/297 objects degraded (9.428%) 2024-09-14T03:43:43.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:43.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1328 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1117 (ObjNum 444 snap 143 seq_num 444) dirty exists 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:1783: finishing write tid 1 to smithi11075562-7 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:1783: finishing write tid 2 to smithi11075562-7 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:1783: finishing write tid 3 to smithi11075562-7 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:1783: finishing write tid 4 to smithi11075562-7 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:1783: finishing write tid 5 to smithi11075562-7 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:1783: finishing write tid 6 to smithi11075562-7 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 586 (ObjNum 568 snap 173 seq_num 568) dirty exists 2024-09-14T03:43:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:1783: left oid 7 (ObjNum 568 snap 173 seq_num 568) 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1782: finishing copy_from racing read to smithi11075562-42 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1782: finishing copy_from to smithi11075562-42 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 692 (ObjNum 455 snap 147 seq_num 455) dirty exists 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1780: done (4 left) 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1781: done (3 left) 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1782: done (2 left) 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1783: done (1 left) 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1784: done (0 left) 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1785: copy_from oid 33 from oid 44 current snap is 173 2024-09-14T03:43:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:1786: snap_create 2024-09-14T03:43:43.774 INFO:tasks.rados.rados.0.smithi110.stdout:1785: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:43:43.774 INFO:tasks.rados.rados.0.smithi110.stdout:1785: finishing copy_from to smithi11075562-33 2024-09-14T03:43:43.775 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 399 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:43:44.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:43 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 28/297 objects degraded (9.428%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:44.149 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:43 smithi195 ceph-mon[82388]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T03:43:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:43 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 28/297 objects degraded (9.428%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:43 smithi110 ceph-mon[88262]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T03:43:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:43 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 28/297 objects degraded (9.428%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:43:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:43 smithi110 ceph-mon[86423]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T03:43:44.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:44.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:43:44.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:44 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:44.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:44.740 INFO:tasks.rados.rados.0.smithi110.stdout:1785: done (1 left) 2024-09-14T03:43:44.740 INFO:tasks.rados.rados.0.smithi110.stdout:1786: done (0 left) 2024-09-14T03:43:44.740 INFO:tasks.rados.rados.0.smithi110.stdout:1787: delete oid 30 current snap is 174 2024-09-14T03:43:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:1787: done (0 left) 2024-09-14T03:43:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:1788: rollback oid 41 current snap is 174 2024-09-14T03:43:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 173 2024-09-14T03:43:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:1789: read oid 42 snap -1 2024-09-14T03:43:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:1789: expect (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:43:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 174 2024-09-14T03:43:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:1790: seq_num 570 ranges {3219456=753664,3973120=548864,4521984=540672,5062656=516096} 2024-09-14T03:43:44.767 INFO:tasks.rados.rados.0.smithi110.stdout:1790: writing smithi11075562-35 from 3219456 to 3973120 tid 1 2024-09-14T03:43:44.770 INFO:tasks.rados.rados.0.smithi110.stdout:1790: writing smithi11075562-35 from 3973120 to 4521984 tid 2 2024-09-14T03:43:44.772 INFO:tasks.rados.rados.0.smithi110.stdout:1790: writing smithi11075562-35 from 4521984 to 5062656 tid 3 2024-09-14T03:43:44.775 INFO:tasks.rados.rados.0.smithi110.stdout:1790: writing smithi11075562-35 from 5062656 to 5578752 tid 4 2024-09-14T03:43:44.776 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 174 2024-09-14T03:43:44.776 INFO:tasks.rados.rados.0.smithi110.stdout:1791: seq_num 571 ranges {1957888=770048,2727936=483328,3211264=753664,3964928=368640} 2024-09-14T03:43:44.791 INFO:tasks.rados.rados.0.smithi110.stdout:1791: writing smithi11075562-29 from 1957888 to 2727936 tid 1 2024-09-14T03:43:44.793 INFO:tasks.rados.rados.0.smithi110.stdout:1791: writing smithi11075562-29 from 2727936 to 3211264 tid 2 2024-09-14T03:43:44.798 INFO:tasks.rados.rados.0.smithi110.stdout:1791: writing smithi11075562-29 from 3211264 to 3964928 tid 3 2024-09-14T03:43:44.800 INFO:tasks.rados.rados.0.smithi110.stdout:1791: writing smithi11075562-29 from 3964928 to 4333568 tid 4 2024-09-14T03:43:44.800 INFO:tasks.rados.rados.0.smithi110.stdout:1792: read oid 4 snap -1 2024-09-14T03:43:44.800 INFO:tasks.rados.rados.0.smithi110.stdout:1792: expect (ObjNum 536 snap 166 seq_num 536) 2024-09-14T03:43:44.801 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 174 2024-09-14T03:43:44.801 INFO:tasks.rados.rados.0.smithi110.stdout:1788: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:43:44.801 INFO:tasks.rados.rados.0.smithi110.stdout:1788: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:43:44.801 INFO:tasks.rados.rados.0.smithi110.stdout:1793: seq_num 572 ranges {0=786432,786432=770048,1556480=557056,2113536=278528} 2024-09-14T03:43:44.805 INFO:tasks.rados.rados.0.smithi110.stdout:1793: writing smithi11075562-1 from 0 to 786432 tid 1 2024-09-14T03:43:44.810 INFO:tasks.rados.rados.0.smithi110.stdout:1793: writing smithi11075562-1 from 786432 to 1556480 tid 2 2024-09-14T03:43:44.813 INFO:tasks.rados.rados.0.smithi110.stdout:1793: writing smithi11075562-1 from 1556480 to 2113536 tid 3 2024-09-14T03:43:44.815 INFO:tasks.rados.rados.0.smithi110.stdout:1793: writing smithi11075562-1 from 2113536 to 2392064 tid 4 2024-09-14T03:43:44.815 INFO:tasks.rados.rados.0.smithi110.stdout:1794: setattr oid 12 current snap is 174 2024-09-14T03:43:44.815 INFO:tasks.rados.rados.0.smithi110.stdout:1788: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:43:44.815 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1048 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:43:44.815 INFO:tasks.rados.rados.0.smithi110.stdout:1789: expect (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:43:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:44 smithi110 ceph-mon[86423]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T03:43:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:44 smithi110 ceph-mon[86423]: pgmap v265: 121 pgs: 1 active+recovering+degraded, 1 active+clean+snaptrim_wait, 5 peering, 114 active+clean; 193 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.2 MiB/s wr, 16 op/s; 4/219 objects degraded (1.826%); 7.0 MiB/s, 2 objects/s recovering 2024-09-14T03:43:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:44 smithi110 ceph-mon[88262]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T03:43:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:44 smithi110 ceph-mon[88262]: pgmap v265: 121 pgs: 1 active+recovering+degraded, 1 active+clean+snaptrim_wait, 5 peering, 114 active+clean; 193 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.2 MiB/s wr, 16 op/s; 4/219 objects degraded (1.826%); 7.0 MiB/s, 2 objects/s recovering 2024-09-14T03:43:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:44 smithi195 ceph-mon[82388]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T03:43:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:44 smithi195 ceph-mon[82388]: pgmap v265: 121 pgs: 1 active+recovering+degraded, 1 active+clean+snaptrim_wait, 5 peering, 114 active+clean; 193 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.2 MiB/s wr, 16 op/s; 4/219 objects degraded (1.826%); 7.0 MiB/s, 2 objects/s recovering 2024-09-14T03:43:45.355 INFO:tasks.rados.rados.0.smithi110.stdout:1790: finishing write tid 1 to smithi11075562-35 2024-09-14T03:43:45.355 INFO:tasks.rados.rados.0.smithi110.stdout:1790: finishing write tid 2 to smithi11075562-35 2024-09-14T03:43:45.355 INFO:tasks.rados.rados.0.smithi110.stdout:1790: finishing write tid 3 to smithi11075562-35 2024-09-14T03:43:45.355 INFO:tasks.rados.rados.0.smithi110.stdout:1790: finishing write tid 4 to smithi11075562-35 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1790: finishing write tid 5 to smithi11075562-35 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1790: finishing write tid 6 to smithi11075562-35 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1145 (ObjNum 570 snap 174 seq_num 570) dirty exists 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1790: left oid 35 (ObjNum 570 snap 174 seq_num 570) 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1791: finishing write tid 1 to smithi11075562-29 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1791: finishing write tid 2 to smithi11075562-29 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1791: finishing write tid 3 to smithi11075562-29 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1791: finishing write tid 4 to smithi11075562-29 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1791: finishing write tid 5 to smithi11075562-29 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1791: finishing write tid 6 to smithi11075562-29 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 594 (ObjNum 571 snap 174 seq_num 571) dirty exists 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1791: left oid 29 (ObjNum 571 snap 174 seq_num 571) 2024-09-14T03:43:45.356 INFO:tasks.rados.rados.0.smithi110.stdout:1792: expect (ObjNum 536 snap 166 seq_num 536) 2024-09-14T03:43:45.400 INFO:tasks.rados.rados.0.smithi110.stdout:1793: finishing write tid 1 to smithi11075562-1 2024-09-14T03:43:45.400 INFO:tasks.rados.rados.0.smithi110.stdout:1793: finishing write tid 2 to smithi11075562-1 2024-09-14T03:43:45.400 INFO:tasks.rados.rados.0.smithi110.stdout:1793: finishing write tid 3 to smithi11075562-1 2024-09-14T03:43:45.400 INFO:tasks.rados.rados.0.smithi110.stdout:1793: finishing write tid 4 to smithi11075562-1 2024-09-14T03:43:45.400 INFO:tasks.rados.rados.0.smithi110.stdout:1793: finishing write tid 5 to smithi11075562-1 2024-09-14T03:43:45.400 INFO:tasks.rados.rados.0.smithi110.stdout:1788: done (6 left) 2024-09-14T03:43:45.400 INFO:tasks.rados.rados.0.smithi110.stdout:1789: done (5 left) 2024-09-14T03:43:45.400 INFO:tasks.rados.rados.0.smithi110.stdout:1790: done (4 left) 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:1791: done (3 left) 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:1792: done (2 left) 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:1795: rmattr oid 18 current snap is 174 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:1793: finishing write tid 6 to smithi11075562-1 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1150 (ObjNum 572 snap 174 seq_num 572) dirty exists 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:1793: left oid 1 (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:1793: done (2 left) 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:1796: snap_create 2024-09-14T03:43:45.401 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1332 (ObjNum 539 snap 166 seq_num 539) dirty exists 2024-09-14T03:43:45.402 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1039 (ObjNum 515 snap 160 seq_num 515) dirty exists 2024-09-14T03:43:45.743 INFO:tasks.rados.rados.0.smithi110.stdout:1794: done (2 left) 2024-09-14T03:43:45.743 INFO:tasks.rados.rados.0.smithi110.stdout:1795: done (1 left) 2024-09-14T03:43:45.743 INFO:tasks.rados.rados.0.smithi110.stdout:1796: done (0 left) 2024-09-14T03:43:45.743 INFO:tasks.rados.rados.0.smithi110.stdout:1797: snap_remove snap 173 2024-09-14T03:43:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:45 smithi110 ceph-mon[88262]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T03:43:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:45 smithi110 ceph-mon[86423]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T03:43:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:45 smithi195 ceph-mon[82388]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T03:43:46.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:46.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:46.754 INFO:tasks.rados.rados.0.smithi110.stdout:1797: done (0 left) 2024-09-14T03:43:46.754 INFO:tasks.rados.rados.0.smithi110.stdout:1798: delete oid 49 current snap is 175 2024-09-14T03:43:46.760 INFO:tasks.rados.rados.0.smithi110.stdout:1798: done (0 left) 2024-09-14T03:43:46.760 INFO:tasks.rados.rados.0.smithi110.stdout:1799: delete oid 40 current snap is 175 2024-09-14T03:43:46.763 INFO:tasks.rados.rados.0.smithi110.stdout:1799: done (0 left) 2024-09-14T03:43:46.763 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 175 2024-09-14T03:43:46.763 INFO:tasks.rados.rados.0.smithi110.stdout:1800: seq_num 575 ranges {2818048=32768} 2024-09-14T03:43:46.779 INFO:tasks.rados.rados.0.smithi110.stdout:1800: writing smithi11075562-15 from 2818048 to 2850816 tid 1 2024-09-14T03:43:46.779 INFO:tasks.rados.rados.0.smithi110.stdout:1801: delete oid 41 current snap is 175 2024-09-14T03:43:46.782 INFO:tasks.rados.rados.0.smithi110.stdout:1801: done (1 left) 2024-09-14T03:43:46.782 INFO:tasks.rados.rados.0.smithi110.stdout:1802: read oid 43 snap -1 2024-09-14T03:43:46.782 INFO:tasks.rados.rados.0.smithi110.stdout:1802: expect (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:43:46.782 INFO:tasks.rados.rados.0.smithi110.stdout:1803: read oid 50 snap -1 2024-09-14T03:43:46.782 INFO:tasks.rados.rados.0.smithi110.stdout:1803: expect (ObjNum 566 snap 173 seq_num 566) 2024-09-14T03:43:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:1804: rollback oid 41 current snap is 175 2024-09-14T03:43:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 174 2024-09-14T03:43:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:1800: finishing write tid 1 to smithi11075562-15 2024-09-14T03:43:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:1800: finishing write tid 2 to smithi11075562-15 2024-09-14T03:43:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:1805: rollback oid 29 current snap is 175 2024-09-14T03:43:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 29 to 174 2024-09-14T03:43:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 175 2024-09-14T03:43:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:1806: seq_num 577 ranges {6365184=65536} 2024-09-14T03:43:46.817 INFO:tasks.rados.rados.0.smithi110.stdout:1806: writing smithi11075562-25 from 6365184 to 6430720 tid 1 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:1807: snap_create 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:1800: finishing write tid 3 to smithi11075562-15 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1123 (ObjNum 575 snap 175 seq_num 575) dirty exists 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:1800: left oid 15 (ObjNum 575 snap 175 seq_num 575) 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:1805: finishing rollback tid 0 to smithi11075562-29 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:1805: finishing rollback tid 1 to smithi11075562-29 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:1805: finishing rollback tid 2 to smithi11075562-29 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 600 (ObjNum 571 snap 174 seq_num 571) dirty exists 2024-09-14T03:43:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:1803: expect (ObjNum 566 snap 173 seq_num 566) 2024-09-14T03:43:46.875 INFO:tasks.rados.rados.0.smithi110.stdout:1804: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:43:46.875 INFO:tasks.rados.rados.0.smithi110.stdout:1804: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:43:46.875 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1054 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:43:46.875 INFO:tasks.rados.rados.0.smithi110.stdout:1802: expect (ObjNum 413 snap 134 seq_num 413) 2024-09-14T03:43:47.019 INFO:tasks.rados.rados.0.smithi110.stdout:1806: finishing write tid 1 to smithi11075562-25 2024-09-14T03:43:47.019 INFO:tasks.rados.rados.0.smithi110.stdout:1806: finishing write tid 2 to smithi11075562-25 2024-09-14T03:43:47.019 INFO:tasks.rados.rados.0.smithi110.stdout:1806: finishing write tid 3 to smithi11075562-25 2024-09-14T03:43:47.019 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1337 (ObjNum 577 snap 175 seq_num 577) dirty exists 2024-09-14T03:43:47.019 INFO:tasks.rados.rados.0.smithi110.stdout:1806: left oid 25 (ObjNum 577 snap 175 seq_num 577) 2024-09-14T03:43:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:46 smithi110 ceph-mon[88262]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T03:43:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:46 smithi110 ceph-mon[88262]: pgmap v268: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 192 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 7.3 MiB/s wr, 24 op/s; 13 MiB/s, 3 objects/s recovering 2024-09-14T03:43:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:46 smithi110 ceph-mon[86423]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T03:43:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:46 smithi110 ceph-mon[86423]: pgmap v268: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 192 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 7.3 MiB/s wr, 24 op/s; 13 MiB/s, 3 objects/s recovering 2024-09-14T03:43:47.208 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:46 smithi195 ceph-mon[82388]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T03:43:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:46 smithi195 ceph-mon[82388]: pgmap v268: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 192 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 7.3 MiB/s wr, 24 op/s; 13 MiB/s, 3 objects/s recovering 2024-09-14T03:43:47.756 INFO:tasks.rados.rados.0.smithi110.stdout:1800: done (6 left) 2024-09-14T03:43:47.756 INFO:tasks.rados.rados.0.smithi110.stdout:1802: done (5 left) 2024-09-14T03:43:47.756 INFO:tasks.rados.rados.0.smithi110.stdout:1803: done (4 left) 2024-09-14T03:43:47.756 INFO:tasks.rados.rados.0.smithi110.stdout:1804: done (3 left) 2024-09-14T03:43:47.756 INFO:tasks.rados.rados.0.smithi110.stdout:1805: done (2 left) 2024-09-14T03:43:47.756 INFO:tasks.rados.rados.0.smithi110.stdout:1806: done (1 left) 2024-09-14T03:43:47.757 INFO:tasks.rados.rados.0.smithi110.stdout:1807: done (0 left) 2024-09-14T03:43:47.757 INFO:tasks.rados.rados.0.smithi110.stdout:1808: rollback oid 13 current snap is 176 2024-09-14T03:43:47.757 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 175 2024-09-14T03:43:47.757 INFO:tasks.rados.rados.0.smithi110.stdout:1809: read oid 46 snap -1 2024-09-14T03:43:47.757 INFO:tasks.rados.rados.0.smithi110.stdout:1809: expect (ObjNum 526 snap 164 seq_num 526) 2024-09-14T03:43:47.757 INFO:tasks.rados.rados.0.smithi110.stdout:1810: rmattr oid 9 current snap is 176 2024-09-14T03:43:47.757 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 176 2024-09-14T03:43:47.757 INFO:tasks.rados.rados.0.smithi110.stdout:1811: seq_num 578 ranges {1785856=81920} 2024-09-14T03:43:47.766 INFO:tasks.rados.rados.0.smithi110.stdout:1811: writing smithi11075562-18 from 1785856 to 1867776 tid 1 2024-09-14T03:43:47.766 INFO:tasks.rados.rados.0.smithi110.stdout:1812: delete oid 26 current snap is 176 2024-09-14T03:43:47.766 INFO:tasks.rados.rados.0.smithi110.stdout:1808: finishing rollback tid 0 to smithi11075562-13 2024-09-14T03:43:47.766 INFO:tasks.rados.rados.0.smithi110.stdout:1808: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:43:47.767 INFO:tasks.rados.rados.0.smithi110.stdout:1808: finishing rollback tid 2 to smithi11075562-13 2024-09-14T03:43:47.767 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1158 (ObjNum 550 snap 167 seq_num 550) dirty exists 2024-09-14T03:43:47.769 INFO:tasks.rados.rados.0.smithi110.stdout:1808: done (4 left) 2024-09-14T03:43:47.769 INFO:tasks.rados.rados.0.smithi110.stdout:1812: done (3 left) 2024-09-14T03:43:47.769 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 176 2024-09-14T03:43:47.769 INFO:tasks.rados.rados.0.smithi110.stdout:1813: seq_num 580 ranges {0=114688} 2024-09-14T03:43:47.769 INFO:tasks.rados.rados.0.smithi110.stdout:1813: writing smithi11075562-32 from 0 to 114688 tid 1 2024-09-14T03:43:47.769 INFO:tasks.rados.rados.0.smithi110.stdout:1814: snap_create 2024-09-14T03:43:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:1813: finishing write tid 1 to smithi11075562-32 2024-09-14T03:43:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:1813: finishing write tid 2 to smithi11075562-32 2024-09-14T03:43:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:1813: finishing write tid 3 to smithi11075562-32 2024-09-14T03:43:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1125 (ObjNum 580 snap 176 seq_num 580) dirty exists 2024-09-14T03:43:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:1813: left oid 32 (ObjNum 580 snap 176 seq_num 580) 2024-09-14T03:43:47.782 INFO:tasks.rados.rados.0.smithi110.stdout:1811: finishing write tid 1 to smithi11075562-18 2024-09-14T03:43:47.788 INFO:tasks.rados.rados.0.smithi110.stdout:1811: finishing write tid 2 to smithi11075562-18 2024-09-14T03:43:47.790 INFO:tasks.rados.rados.0.smithi110.stdout:1811: finishing write tid 3 to smithi11075562-18 2024-09-14T03:43:47.790 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1046 (ObjNum 578 snap 176 seq_num 578) dirty exists 2024-09-14T03:43:47.790 INFO:tasks.rados.rados.0.smithi110.stdout:1811: left oid 18 (ObjNum 578 snap 176 seq_num 578) 2024-09-14T03:43:47.805 INFO:tasks.rados.rados.0.smithi110.stdout:1809: expect (ObjNum 526 snap 164 seq_num 526) 2024-09-14T03:43:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:47 smithi110 ceph-mon[86423]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/219 objects degraded (1.826%), 1 pg degraded) 2024-09-14T03:43:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:47 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:43:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:47 smithi110 ceph-mon[86423]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T03:43:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:47 smithi110 ceph-mon[88262]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/219 objects degraded (1.826%), 1 pg degraded) 2024-09-14T03:43:48.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:47 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:43:48.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:47 smithi110 ceph-mon[88262]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T03:43:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:47 smithi195 ceph-mon[82388]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/219 objects degraded (1.826%), 1 pg degraded) 2024-09-14T03:43:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:47 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:43:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:47 smithi195 ceph-mon[82388]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T03:43:48.396 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1036 (ObjNum 543 snap 166 seq_num 543) dirty exists 2024-09-14T03:43:48.705 INFO:tasks.rados.rados.0.smithi110.stdout:1809: done (4 left) 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1810: done (3 left) 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1811: done (2 left) 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1813: done (1 left) 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1814: done (0 left) 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1815: rollback oid 21 current snap is 177 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 176 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1816: read oid 13 snap 175 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1816: expect (ObjNum 550 snap 167 seq_num 550) 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1817: copy_from oid 4 from oid 1 current snap is 177 2024-09-14T03:43:48.706 INFO:tasks.rados.rados.0.smithi110.stdout:1818: snap_create 2024-09-14T03:43:48.709 INFO:tasks.rados.rados.0.smithi110.stdout:1815: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:43:48.709 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 0 (ObjNum 65 snap 0 seq_num 1482778202) dirty dne 2024-09-14T03:43:48.736 INFO:tasks.rados.rados.0.smithi110.stdout:1816: expect (ObjNum 550 snap 167 seq_num 550) 2024-09-14T03:43:49.019 INFO:tasks.rados.rados.0.smithi110.stdout:1817: finishing copy_from racing read to smithi11075562-4 2024-09-14T03:43:49.019 INFO:tasks.rados.rados.0.smithi110.stdout:1817: finishing copy_from to smithi11075562-4 2024-09-14T03:43:49.019 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1150 (ObjNum 572 snap 174 seq_num 572) dirty exists 2024-09-14T03:43:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:48 smithi110 ceph-mon[86423]: pgmap v271: 121 pgs: 121 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 4.7 MiB/s wr, 27 op/s; 6.2 MiB/s, 0 objects/s recovering 2024-09-14T03:43:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:48 smithi110 ceph-mon[86423]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T03:43:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:48 smithi110 ceph-mon[86423]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T03:43:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:48 smithi110 ceph-mon[88262]: pgmap v271: 121 pgs: 121 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 4.7 MiB/s wr, 27 op/s; 6.2 MiB/s, 0 objects/s recovering 2024-09-14T03:43:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:48 smithi110 ceph-mon[88262]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T03:43:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:48 smithi110 ceph-mon[88262]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T03:43:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:48 smithi195 ceph-mon[82388]: pgmap v271: 121 pgs: 121 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 4.7 MiB/s wr, 27 op/s; 6.2 MiB/s, 0 objects/s recovering 2024-09-14T03:43:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:48 smithi195 ceph-mon[82388]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T03:43:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:48 smithi195 ceph-mon[82388]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T03:43:49.707 INFO:tasks.rados.rados.0.smithi110.stdout:1815: done (3 left) 2024-09-14T03:43:49.707 INFO:tasks.rados.rados.0.smithi110.stdout:1816: done (2 left) 2024-09-14T03:43:49.707 INFO:tasks.rados.rados.0.smithi110.stdout:1817: done (1 left) 2024-09-14T03:43:49.707 INFO:tasks.rados.rados.0.smithi110.stdout:1818: done (0 left) 2024-09-14T03:43:49.707 INFO:tasks.rados.rados.0.smithi110.stdout:1819: rollback oid 1 current snap is 178 2024-09-14T03:43:49.707 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 1 to 177 2024-09-14T03:43:49.707 INFO:tasks.rados.rados.0.smithi110.stdout:1820: read oid 38 snap -1 2024-09-14T03:43:49.708 INFO:tasks.rados.rados.0.smithi110.stdout:1820: expect (ObjNum 519 snap 163 seq_num 519) 2024-09-14T03:43:49.708 INFO:tasks.rados.rados.0.smithi110.stdout:1821: rollback oid 18 current snap is 178 2024-09-14T03:43:49.708 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 18 to 176 2024-09-14T03:43:49.708 INFO:tasks.rados.rados.0.smithi110.stdout:1822: rollback oid 27 current snap is 178 2024-09-14T03:43:49.708 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 174 2024-09-14T03:43:49.708 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 178 2024-09-14T03:43:49.708 INFO:tasks.rados.rados.0.smithi110.stdout:1823: seq_num 581 ranges {5578752=139264} 2024-09-14T03:43:49.738 INFO:tasks.rados.rados.0.smithi110.stdout:1823: writing smithi11075562-35 from 5578752 to 5718016 tid 1 2024-09-14T03:43:49.738 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 178 2024-09-14T03:43:49.738 INFO:tasks.rados.rados.0.smithi110.stdout:1821: finishing rollback tid 0 to smithi11075562-18 2024-09-14T03:43:49.739 INFO:tasks.rados.rados.0.smithi110.stdout:1824: seq_num 582 ranges {2449408=155648} 2024-09-14T03:43:49.753 INFO:tasks.rados.rados.0.smithi110.stdout:1824: writing smithi11075562-8 from 2449408 to 2605056 tid 1 2024-09-14T03:43:49.753 INFO:tasks.rados.rados.0.smithi110.stdout:1825: copy_from oid 14 from oid 21 current snap is 178 2024-09-14T03:43:49.753 INFO:tasks.rados.rados.0.smithi110.stdout:1819: finishing rollback tid 0 to smithi11075562-1 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1819: finishing rollback tid 1 to smithi11075562-1 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1821: finishing rollback tid 1 to smithi11075562-18 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1819: finishing rollback tid 2 to smithi11075562-1 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1162 (ObjNum 572 snap 174 seq_num 572) dirty exists 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1821: finishing rollback tid 2 to smithi11075562-18 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1050 (ObjNum 578 snap 176 seq_num 578) dirty exists 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1822: finishing rollback tid 0 to smithi11075562-27 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1822: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1822: finishing rollback tid 2 to smithi11075562-27 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 685 (ObjNum 464 snap 150 seq_num 464) dirty exists 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1819: done (6 left) 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1821: done (5 left) 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1822: done (4 left) 2024-09-14T03:43:49.754 INFO:tasks.rados.rados.0.smithi110.stdout:1826: snap_remove snap 174 2024-09-14T03:43:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:50.718 INFO:tasks.rados.rados.0.smithi110.stdout:1826: done (4 left) 2024-09-14T03:43:50.718 INFO:tasks.rados.rados.0.smithi110.stdout:1827: read oid 49 snap -1 2024-09-14T03:43:50.718 INFO:tasks.rados.rados.0.smithi110.stdout:1827: expect deleted 2024-09-14T03:43:50.718 INFO:tasks.rados.rados.0.smithi110.stdout:1828: read oid 27 snap 176 2024-09-14T03:43:50.718 INFO:tasks.rados.rados.0.smithi110.stdout:1828: expect (ObjNum 464 snap 150 seq_num 464) 2024-09-14T03:43:50.718 INFO:tasks.rados.rados.0.smithi110.stdout:1829: read oid 10 snap 175 2024-09-14T03:43:50.718 INFO:tasks.rados.rados.0.smithi110.stdout:1829: expect (ObjNum 500 snap 156 seq_num 500) 2024-09-14T03:43:50.718 INFO:tasks.rados.rados.0.smithi110.stdout:1830: read oid 18 snap -1 2024-09-14T03:43:50.719 INFO:tasks.rados.rados.0.smithi110.stdout:1830: expect (ObjNum 578 snap 176 seq_num 578) 2024-09-14T03:43:50.719 INFO:tasks.rados.rados.0.smithi110.stdout:1831: snap_create 2024-09-14T03:43:50.719 INFO:tasks.rados.rados.0.smithi110.stdout:1820: expect (ObjNum 519 snap 163 seq_num 519) 2024-09-14T03:43:50.923 INFO:tasks.rados.rados.0.smithi110.stdout:1823: finishing write tid 1 to smithi11075562-35 2024-09-14T03:43:50.923 INFO:tasks.rados.rados.0.smithi110.stdout:1823: finishing write tid 2 to smithi11075562-35 2024-09-14T03:43:50.923 INFO:tasks.rados.rados.0.smithi110.stdout:1823: finishing write tid 3 to smithi11075562-35 2024-09-14T03:43:50.923 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1165 (ObjNum 581 snap 178 seq_num 581) dirty exists 2024-09-14T03:43:50.923 INFO:tasks.rados.rados.0.smithi110.stdout:1823: left oid 35 (ObjNum 581 snap 178 seq_num 581) 2024-09-14T03:43:50.923 INFO:tasks.rados.rados.0.smithi110.stdout:1825: finishing copy_from to smithi11075562-14 2024-09-14T03:43:50.923 INFO:tasks.rados.rados.0.smithi110.stdout:1825: got expected ENOENT (src dne) 2024-09-14T03:43:50.924 INFO:tasks.rados.rados.0.smithi110.stdout:1825: finishing copy_from racing read to smithi11075562-14 2024-09-14T03:43:50.924 INFO:tasks.rados.rados.0.smithi110.stdout:1824: finishing write tid 1 to smithi11075562-8 2024-09-14T03:43:50.924 INFO:tasks.rados.rados.0.smithi110.stdout:1824: finishing write tid 2 to smithi11075562-8 2024-09-14T03:43:50.924 INFO:tasks.rados.rados.0.smithi110.stdout:1824: finishing write tid 3 to smithi11075562-8 2024-09-14T03:43:50.924 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1128 (ObjNum 582 snap 178 seq_num 582) dirty exists 2024-09-14T03:43:50.924 INFO:tasks.rados.rados.0.smithi110.stdout:1824: left oid 8 (ObjNum 582 snap 178 seq_num 582) 2024-09-14T03:43:50.924 INFO:tasks.rados.rados.0.smithi110.stdout:1830: expect (ObjNum 578 snap 176 seq_num 578) 2024-09-14T03:43:51.006 INFO:tasks.rados.rados.0.smithi110.stdout:1829: expect (ObjNum 500 snap 156 seq_num 500) 2024-09-14T03:43:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:50 smithi110 ceph-mon[88262]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T03:43:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:50 smithi110 ceph-mon[88262]: pgmap v274: 121 pgs: 121 active+clean; 220 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.0 MiB/s wr, 26 op/s 2024-09-14T03:43:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:50 smithi110 ceph-mon[86423]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T03:43:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:50 smithi110 ceph-mon[86423]: pgmap v274: 121 pgs: 121 active+clean; 220 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.0 MiB/s wr, 26 op/s 2024-09-14T03:43:51.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:50 smithi195 ceph-mon[82388]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T03:43:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:50 smithi195 ceph-mon[82388]: pgmap v274: 121 pgs: 121 active+clean; 220 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.0 MiB/s wr, 26 op/s 2024-09-14T03:43:51.640 INFO:tasks.rados.rados.0.smithi110.stdout:1828: expect (ObjNum 464 snap 150 seq_num 464) 2024-09-14T03:43:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:51 smithi110 ceph-mon[88262]: osdmap e432: 8 total, 8 up, 8 in 2024-09-14T03:43:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:51 smithi110 ceph-mon[86423]: osdmap e432: 8 total, 8 up, 8 in 2024-09-14T03:43:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:43:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:43:51] "GET /metrics HTTP/1.1" 200 36109 "" "Prometheus/2.51.0" 2024-09-14T03:43:52.181 INFO:tasks.rados.rados.0.smithi110.stdout:1820: done (8 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1823: done (7 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1824: done (6 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1825: done (5 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1827: done (4 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1828: done (3 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1829: done (2 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1830: done (1 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1831: done (0 left) 2024-09-14T03:43:52.182 INFO:tasks.rados.rados.0.smithi110.stdout:1832: delete oid 46 current snap is 179 2024-09-14T03:43:52.187 INFO:tasks.rados.rados.0.smithi110.stdout:1832: done (0 left) 2024-09-14T03:43:52.187 INFO:tasks.rados.rados.0.smithi110.stdout:1833: rollback oid 6 current snap is 179 2024-09-14T03:43:52.187 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 6 to 175 2024-09-14T03:43:52.187 INFO:tasks.rados.rados.0.smithi110.stdout:1834: read oid 17 snap -1 2024-09-14T03:43:52.187 INFO:tasks.rados.rados.0.smithi110.stdout:1834: expect (ObjNum 551 snap 167 seq_num 551) 2024-09-14T03:43:52.187 INFO:tasks.rados.rados.0.smithi110.stdout:1835: snap_create 2024-09-14T03:43:52.190 INFO:tasks.rados.rados.0.smithi110.stdout:1833: finishing rollback tid 1 to smithi11075562-6 2024-09-14T03:43:52.190 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 0 (ObjNum 97 snap 0 seq_num 64481712) dirty dne 2024-09-14T03:43:52.197 INFO:tasks.rados.rados.0.smithi110.stdout:1834: expect (ObjNum 551 snap 167 seq_num 551) 2024-09-14T03:43:52.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:51 smithi195 ceph-mon[82388]: osdmap e432: 8 total, 8 up, 8 in 2024-09-14T03:43:53.086 INFO:tasks.rados.rados.0.smithi110.stdout:1833: done (2 left) 2024-09-14T03:43:53.086 INFO:tasks.rados.rados.0.smithi110.stdout:1834: done (1 left) 2024-09-14T03:43:53.086 INFO:tasks.rados.rados.0.smithi110.stdout:1835: done (0 left) 2024-09-14T03:43:53.086 INFO:tasks.rados.rados.0.smithi110.stdout:1836: rollback oid 5 current snap is 180 2024-09-14T03:43:53.086 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 5 to 179 2024-09-14T03:43:53.086 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 180 2024-09-14T03:43:53.087 INFO:tasks.rados.rados.0.smithi110.stdout:1837: seq_num 584 ranges {0=188416} 2024-09-14T03:43:53.087 INFO:tasks.rados.rados.0.smithi110.stdout:1837: writing smithi11075562-46 from 0 to 188416 tid 1 2024-09-14T03:43:53.087 INFO:tasks.rados.rados.0.smithi110.stdout:1838: snap_create 2024-09-14T03:43:53.091 INFO:tasks.rados.rados.0.smithi110.stdout:1837: finishing write tid 1 to smithi11075562-46 2024-09-14T03:43:53.091 INFO:tasks.rados.rados.0.smithi110.stdout:1837: finishing write tid 2 to smithi11075562-46 2024-09-14T03:43:53.093 INFO:tasks.rados.rados.0.smithi110.stdout:1837: finishing write tid 3 to smithi11075562-46 2024-09-14T03:43:53.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1039 (ObjNum 584 snap 180 seq_num 584) dirty exists 2024-09-14T03:43:53.093 INFO:tasks.rados.rados.0.smithi110.stdout:1837: left oid 46 (ObjNum 584 snap 180 seq_num 584) 2024-09-14T03:43:53.094 INFO:tasks.rados.rados.0.smithi110.stdout:1836: finishing rollback tid 0 to smithi11075562-5 2024-09-14T03:43:53.094 INFO:tasks.rados.rados.0.smithi110.stdout:1836: finishing rollback tid 1 to smithi11075562-5 2024-09-14T03:43:53.094 INFO:tasks.rados.rados.0.smithi110.stdout:1836: finishing rollback tid 2 to smithi11075562-5 2024-09-14T03:43:53.095 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1343 (ObjNum 542 snap 166 seq_num 542) dirty exists 2024-09-14T03:43:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:52 smithi110 ceph-mon[86423]: osdmap e433: 8 total, 8 up, 8 in 2024-09-14T03:43:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:52 smithi110 ceph-mon[86423]: pgmap v277: 121 pgs: 121 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.0 MiB/s wr, 10 op/s 2024-09-14T03:43:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:52 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:52 smithi110 ceph-mon[88262]: osdmap e433: 8 total, 8 up, 8 in 2024-09-14T03:43:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:52 smithi110 ceph-mon[88262]: pgmap v277: 121 pgs: 121 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.0 MiB/s wr, 10 op/s 2024-09-14T03:43:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:52 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:52 smithi195 ceph-mon[82388]: osdmap e433: 8 total, 8 up, 8 in 2024-09-14T03:43:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:52 smithi195 ceph-mon[82388]: pgmap v277: 121 pgs: 121 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.0 MiB/s wr, 10 op/s 2024-09-14T03:43:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:52 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1836: done (2 left) 2024-09-14T03:43:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1837: done (1 left) 2024-09-14T03:43:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1838: done (0 left) 2024-09-14T03:43:53.706 INFO:tasks.rados.rados.0.smithi110.stdout:1839: snap_remove snap 180 2024-09-14T03:43:54.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:54 smithi110 ceph-mon[86423]: osdmap e434: 8 total, 8 up, 8 in 2024-09-14T03:43:54.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:54 smithi110 ceph-mon[86423]: osdmap e435: 8 total, 8 up, 8 in 2024-09-14T03:43:54.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:54 smithi110 ceph-mon[86423]: pgmap v280: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 674 KiB/s wr, 16 op/s 2024-09-14T03:43:54.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:54 smithi110 ceph-mon[88262]: osdmap e434: 8 total, 8 up, 8 in 2024-09-14T03:43:54.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:54 smithi110 ceph-mon[88262]: osdmap e435: 8 total, 8 up, 8 in 2024-09-14T03:43:54.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:54 smithi110 ceph-mon[88262]: pgmap v280: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 674 KiB/s wr, 16 op/s 2024-09-14T03:43:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:54 smithi195 ceph-mon[82388]: osdmap e434: 8 total, 8 up, 8 in 2024-09-14T03:43:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:54 smithi195 ceph-mon[82388]: osdmap e435: 8 total, 8 up, 8 in 2024-09-14T03:43:54.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:54 smithi195 ceph-mon[82388]: pgmap v280: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 674 KiB/s wr, 16 op/s 2024-09-14T03:43:54.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:54.712 INFO:tasks.rados.rados.0.smithi110.stdout:1839: done (0 left) 2024-09-14T03:43:54.712 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 181 2024-09-14T03:43:54.712 INFO:tasks.rados.rados.0.smithi110.stdout:1840: seq_num 585 ranges {0=204800} 2024-09-14T03:43:54.713 INFO:tasks.rados.rados.0.smithi110.stdout:1840: writing smithi11075562-49 from 0 to 204800 tid 1 2024-09-14T03:43:54.713 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 181 2024-09-14T03:43:54.713 INFO:tasks.rados.rados.0.smithi110.stdout:1841: seq_num 586 ranges {6430720=221184} 2024-09-14T03:43:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1841: writing smithi11075562-25 from 6430720 to 6651904 tid 1 2024-09-14T03:43:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1842: copy_from oid 30 from oid 13 current snap is 181 2024-09-14T03:43:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1840: finishing write tid 1 to smithi11075562-49 2024-09-14T03:43:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1840: finishing write tid 2 to smithi11075562-49 2024-09-14T03:43:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1840: finishing write tid 3 to smithi11075562-49 2024-09-14T03:43:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1175 (ObjNum 585 snap 181 seq_num 585) dirty exists 2024-09-14T03:43:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1840: left oid 49 (ObjNum 585 snap 181 seq_num 585) 2024-09-14T03:43:54.750 INFO:tasks.rados.rados.0.smithi110.stdout:1840: done (2 left) 2024-09-14T03:43:54.751 INFO:tasks.rados.rados.0.smithi110.stdout:1843: read oid 36 snap -1 2024-09-14T03:43:54.751 INFO:tasks.rados.rados.0.smithi110.stdout:1843: expect (ObjNum 508 snap 160 seq_num 508) 2024-09-14T03:43:54.751 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 181 2024-09-14T03:43:54.751 INFO:tasks.rados.rados.0.smithi110.stdout:1844: seq_num 587 ranges {4005888=237568} 2024-09-14T03:43:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:1844: writing smithi11075562-37 from 4005888 to 4243456 tid 1 2024-09-14T03:43:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 181 2024-09-14T03:43:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:1841: finishing write tid 1 to smithi11075562-25 2024-09-14T03:43:54.776 INFO:tasks.rados.rados.0.smithi110.stdout:1843: expect (ObjNum 508 snap 160 seq_num 508) 2024-09-14T03:43:54.813 INFO:tasks.rados.rados.0.smithi110.stdout:1841: finishing write tid 2 to smithi11075562-25 2024-09-14T03:43:54.813 INFO:tasks.rados.rados.0.smithi110.stdout:1841: finishing write tid 3 to smithi11075562-25 2024-09-14T03:43:54.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1346 (ObjNum 586 snap 181 seq_num 586) dirty exists 2024-09-14T03:43:54.813 INFO:tasks.rados.rados.0.smithi110.stdout:1841: left oid 25 (ObjNum 586 snap 181 seq_num 586) 2024-09-14T03:43:54.813 INFO:tasks.rados.rados.0.smithi110.stdout:1845: seq_num 588 ranges {6807552=253952} 2024-09-14T03:43:54.850 INFO:tasks.rados.rados.0.smithi110.stdout:1845: writing smithi11075562-39 from 6807552 to 7061504 tid 1 2024-09-14T03:43:54.850 INFO:tasks.rados.rados.0.smithi110.stdout:1841: done (4 left) 2024-09-14T03:43:54.850 INFO:tasks.rados.rados.0.smithi110.stdout:1843: done (3 left) 2024-09-14T03:43:54.851 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 181 2024-09-14T03:43:54.851 INFO:tasks.rados.rados.0.smithi110.stdout:1842: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:43:54.851 INFO:tasks.rados.rados.0.smithi110.stdout:1844: finishing write tid 1 to smithi11075562-37 2024-09-14T03:43:54.851 INFO:tasks.rados.rados.0.smithi110.stdout:1846: seq_num 589 ranges {9715712=270336} 2024-09-14T03:43:54.903 INFO:tasks.rados.rados.0.smithi110.stdout:1846: writing smithi11075562-24 from 9715712 to 9986048 tid 1 2024-09-14T03:43:54.903 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 181 2024-09-14T03:43:54.903 INFO:tasks.rados.rados.0.smithi110.stdout:1844: finishing write tid 2 to smithi11075562-37 2024-09-14T03:43:54.904 INFO:tasks.rados.rados.0.smithi110.stdout:1844: finishing write tid 3 to smithi11075562-37 2024-09-14T03:43:54.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1349 (ObjNum 587 snap 181 seq_num 587) dirty exists 2024-09-14T03:43:54.904 INFO:tasks.rados.rados.0.smithi110.stdout:1844: left oid 37 (ObjNum 587 snap 181 seq_num 587) 2024-09-14T03:43:54.904 INFO:tasks.rados.rados.0.smithi110.stdout:1842: finishing copy_from to smithi11075562-30 2024-09-14T03:43:54.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1158 (ObjNum 550 snap 167 seq_num 550) dirty exists 2024-09-14T03:43:54.904 INFO:tasks.rados.rados.0.smithi110.stdout:1847: seq_num 590 ranges {11575296=286720} 2024-09-14T03:43:54.965 INFO:tasks.rados.rados.0.smithi110.stdout:1847: writing smithi11075562-22 from 11575296 to 11862016 tid 1 2024-09-14T03:43:54.966 INFO:tasks.rados.rados.0.smithi110.stdout:1842: done (4 left) 2024-09-14T03:43:54.966 INFO:tasks.rados.rados.0.smithi110.stdout:1844: done (3 left) 2024-09-14T03:43:54.966 INFO:tasks.rados.rados.0.smithi110.stdout:1848: delete oid 40 current snap is 181 2024-09-14T03:43:54.966 INFO:tasks.rados.rados.0.smithi110.stdout:1845: finishing write tid 1 to smithi11075562-39 2024-09-14T03:43:54.966 INFO:tasks.rados.rados.0.smithi110.stdout:1845: finishing write tid 2 to smithi11075562-39 2024-09-14T03:43:54.967 INFO:tasks.rados.rados.0.smithi110.stdout:1845: finishing write tid 3 to smithi11075562-39 2024-09-14T03:43:54.967 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1059 (ObjNum 588 snap 181 seq_num 588) dirty exists 2024-09-14T03:43:54.967 INFO:tasks.rados.rados.0.smithi110.stdout:1845: left oid 39 (ObjNum 588 snap 181 seq_num 588) 2024-09-14T03:43:54.967 INFO:tasks.rados.rados.0.smithi110.stdout:1846: finishing write tid 1 to smithi11075562-24 2024-09-14T03:43:54.967 INFO:tasks.rados.rados.0.smithi110.stdout:1846: finishing write tid 2 to smithi11075562-24 2024-09-14T03:43:54.967 INFO:tasks.rados.rados.0.smithi110.stdout:1846: finishing write tid 3 to smithi11075562-24 2024-09-14T03:43:54.968 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 909 (ObjNum 589 snap 181 seq_num 589) dirty exists 2024-09-14T03:43:54.968 INFO:tasks.rados.rados.0.smithi110.stdout:1846: left oid 24 (ObjNum 589 snap 181 seq_num 589) 2024-09-14T03:43:54.969 INFO:tasks.rados.rados.0.smithi110.stdout:1845: done (3 left) 2024-09-14T03:43:54.969 INFO:tasks.rados.rados.0.smithi110.stdout:1846: done (2 left) 2024-09-14T03:43:54.969 INFO:tasks.rados.rados.0.smithi110.stdout:1848: done (1 left) 2024-09-14T03:43:54.969 INFO:tasks.rados.rados.0.smithi110.stdout:1849: rollback oid 49 current snap is 181 2024-09-14T03:43:54.969 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 176 2024-09-14T03:43:54.970 INFO:tasks.rados.rados.0.smithi110.stdout:1850: snap_remove snap 175 2024-09-14T03:43:55.350 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:43:55.351 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:55 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:43:55.351 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:55 smithi195 ceph-mon[82388]: Upgrade: osd.4 is safe to restart 2024-09-14T03:43:55.351 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:55 smithi195 ceph-mon[82388]: osdmap e436: 8 total, 8 up, 8 in 2024-09-14T03:43:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:43:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:55 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:43:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:55 smithi110 ceph-mon[88262]: Upgrade: osd.4 is safe to restart 2024-09-14T03:43:55.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:55 smithi110 ceph-mon[88262]: osdmap e436: 8 total, 8 up, 8 in 2024-09-14T03:43:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:43:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:55 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:43:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:55 smithi110 ceph-mon[86423]: Upgrade: osd.4 is safe to restart 2024-09-14T03:43:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:55 smithi110 ceph-mon[86423]: osdmap e436: 8 total, 8 up, 8 in 2024-09-14T03:43:55.531 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:43:55.712 INFO:tasks.rados.rados.0.smithi110.stdout:1850: done (2 left) 2024-09-14T03:43:55.712 INFO:tasks.rados.rados.0.smithi110.stdout:1851: copy_from oid 8 from oid 26 current snap is 181 2024-09-14T03:43:55.712 INFO:tasks.rados.rados.0.smithi110.stdout:1847: finishing write tid 1 to smithi11075562-22 2024-09-14T03:43:55.712 INFO:tasks.rados.rados.0.smithi110.stdout:1847: finishing write tid 2 to smithi11075562-22 2024-09-14T03:43:55.712 INFO:tasks.rados.rados.0.smithi110.stdout:1849: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:1847: finishing write tid 3 to smithi11075562-22 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1160 (ObjNum 590 snap 181 seq_num 590) dirty exists 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:1847: left oid 22 (ObjNum 590 snap 181 seq_num 590) 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:1849: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1177 (ObjNum 65 snap 0 seq_num 1751151450) dirty dne 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:1847: done (2 left) 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:1849: done (1 left) 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 181 2024-09-14T03:43:55.713 INFO:tasks.rados.rados.0.smithi110.stdout:1852: seq_num 592 ranges {4243456=319488} 2024-09-14T03:43:55.738 INFO:tasks.rados.rados.0.smithi110.stdout:1852: writing smithi11075562-37 from 4243456 to 4562944 tid 1 2024-09-14T03:43:55.738 INFO:tasks.rados.rados.0.smithi110.stdout:1853: rollback oid 12 current snap is 181 2024-09-14T03:43:55.738 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 176 2024-09-14T03:43:55.738 INFO:tasks.rados.rados.0.smithi110.stdout:1851: finishing copy_from to smithi11075562-8 2024-09-14T03:43:55.738 INFO:tasks.rados.rados.0.smithi110.stdout:1851: got expected ENOENT (src dne) 2024-09-14T03:43:55.738 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 181 2024-09-14T03:43:55.738 INFO:tasks.rados.rados.0.smithi110.stdout:1851: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:43:55.739 INFO:tasks.rados.rados.0.smithi110.stdout:1854: seq_num 593 ranges {0=335872} 2024-09-14T03:43:55.740 INFO:tasks.rados.rados.0.smithi110.stdout:1854: writing smithi11075562-47 from 0 to 335872 tid 1 2024-09-14T03:43:55.740 INFO:tasks.rados.rados.0.smithi110.stdout:1851: done (3 left) 2024-09-14T03:43:55.740 INFO:tasks.rados.rados.0.smithi110.stdout:1855: rollback oid 33 current snap is 181 2024-09-14T03:43:55.740 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 33 to 177 2024-09-14T03:43:55.740 INFO:tasks.rados.rados.0.smithi110.stdout:1856: setattr oid 15 current snap is 181 2024-09-14T03:43:55.742 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 181 2024-09-14T03:43:55.742 INFO:tasks.rados.rados.0.smithi110.stdout:1857: seq_num 594 ranges {4038656=352256} 2024-09-14T03:43:55.766 INFO:tasks.rados.rados.0.smithi110.stdout:1857: writing smithi11075562-43 from 4038656 to 4390912 tid 1 2024-09-14T03:43:55.767 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 181 2024-09-14T03:43:55.767 INFO:tasks.rados.rados.0.smithi110.stdout:1858: seq_num 595 ranges {1867776=368640} 2024-09-14T03:43:55.781 INFO:tasks.rados.rados.0.smithi110.stdout:1858: writing smithi11075562-18 from 1867776 to 2236416 tid 1 2024-09-14T03:43:55.781 INFO:tasks.rados.rados.0.smithi110.stdout:1859: read oid 48 snap -1 2024-09-14T03:43:55.781 INFO:tasks.rados.rados.0.smithi110.stdout:1859: expect (ObjNum 562 snap 171 seq_num 562) 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1860: rollback oid 1 current snap is 181 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 1 to 178 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1855: finishing rollback tid 0 to smithi11075562-33 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1861: rollback oid 39 current snap is 181 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 176 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1852: finishing write tid 1 to smithi11075562-37 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1852: finishing write tid 2 to smithi11075562-37 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1855: finishing rollback tid 1 to smithi11075562-33 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1855: finishing rollback tid 2 to smithi11075562-33 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1042 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1853: finishing rollback tid 0 to smithi11075562-12 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1161 (ObjNum 575 snap 175 seq_num 575) dirty exists 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1853: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:43:55.782 INFO:tasks.rados.rados.0.smithi110.stdout:1852: finishing write tid 3 to smithi11075562-37 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1357 (ObjNum 592 snap 181 seq_num 592) dirty exists 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1852: left oid 37 (ObjNum 592 snap 181 seq_num 592) 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1853: finishing rollback tid 2 to smithi11075562-12 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1361 (ObjNum 539 snap 166 seq_num 539) dirty exists 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1854: finishing write tid 1 to smithi11075562-47 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1854: finishing write tid 2 to smithi11075562-47 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1854: finishing write tid 3 to smithi11075562-47 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1363 (ObjNum 593 snap 181 seq_num 593) dirty exists 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1854: left oid 47 (ObjNum 593 snap 181 seq_num 593) 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1857: finishing write tid 1 to smithi11075562-43 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1857: finishing write tid 2 to smithi11075562-43 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1857: finishing write tid 3 to smithi11075562-43 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1064 (ObjNum 594 snap 181 seq_num 594) dirty exists 2024-09-14T03:43:55.783 INFO:tasks.rados.rados.0.smithi110.stdout:1857: left oid 43 (ObjNum 594 snap 181 seq_num 594) 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1852: done (9 left) 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1853: done (8 left) 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1854: done (7 left) 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1855: done (6 left) 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1856: done (5 left) 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1857: done (4 left) 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1862: read oid 2 snap -1 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1862: expect (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1863: rollback oid 34 current snap is 181 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 178 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1864: snap_create 2024-09-14T03:43:55.784 INFO:tasks.rados.rados.0.smithi110.stdout:1863: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:43:55.785 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 0 (ObjNum 97 snap 0 seq_num 1497023714) dirty dne 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:1861: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:1858: finishing write tid 1 to smithi11075562-18 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:1858: finishing write tid 2 to smithi11075562-18 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:1860: finishing rollback tid 0 to smithi11075562-1 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:1860: finishing rollback tid 1 to smithi11075562-1 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:1860: finishing rollback tid 2 to smithi11075562-1 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1187 (ObjNum 572 snap 174 seq_num 572) dirty exists 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:1858: finishing write tid 3 to smithi11075562-18 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1060 (ObjNum 595 snap 181 seq_num 595) dirty exists 2024-09-14T03:43:55.786 INFO:tasks.rados.rados.0.smithi110.stdout:1858: left oid 18 (ObjNum 595 snap 181 seq_num 595) 2024-09-14T03:43:55.788 INFO:tasks.rados.rados.0.smithi110.stdout:1861: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:43:55.789 INFO:tasks.rados.rados.0.smithi110.stdout:1861: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:43:55.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1067 (ObjNum 389 snap 129 seq_num 389) dirty exists 2024-09-14T03:43:55.802 INFO:tasks.rados.rados.0.smithi110.stdout:1862: expect (ObjNum 455 snap 147 seq_num 455) 2024-09-14T03:43:56.028 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:56.029 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (4m) 20s ago 8m 19.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (4m) 2m ago 8m 78.7M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (10m) 20s ago 10m 20.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (10m) 2m ago 10m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (3m) 2m ago 13m 452M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (4m) 20s ago 14m 525M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (3m) 20s ago 15m 55.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:43:56.126 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (2m) 2m ago 13m 21.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (3m) 20s ago 13m 49.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (4m) 20s ago 8m 9.92M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (4m) 2m ago 8m 9722k - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (2m) 20s ago 13m 218M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (90s) 20s ago 12m 117M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (57s) 20s ago 12m 101M 2480M 19.3.0-4907-ga2782315 8801d2529e43 0cdc58c38711 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (24s) 20s ago 12m 14.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2283ebcb5f10 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (11m) 2m ago 11m 229M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 43ef4aa248ac 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (11m) 2m ago 11m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (10m) 2m ago 10m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (10m) 2m ago 10m 106M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:43:56.127 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (3m) 2m ago 9m 44.2M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:43:56.364 INFO:tasks.rados.rados.0.smithi110.stdout:1859: expect (ObjNum 562 snap 171 seq_num 562) 2024-09-14T03:43:56.468 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 4, 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 4 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:43:56.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:43:56.470 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:43:56.470 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 6, 2024-09-14T03:43:56.470 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 9 2024-09-14T03:43:56.470 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:43:56.470 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "9/20 daemons upgraded", 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:43:56.766 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:43:56.767 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:43:56.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[88262]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:56.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[88262]: osdmap e437: 8 total, 8 up, 8 in 2024-09-14T03:43:56.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[88262]: pgmap v283: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 92 KiB/s wr, 15 op/s 2024-09-14T03:43:56.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:56.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:43:56.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:56.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4009832503' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:56.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[86423]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:56.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[86423]: osdmap e437: 8 total, 8 up, 8 in 2024-09-14T03:43:56.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[86423]: pgmap v283: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 92 KiB/s wr, 15 op/s 2024-09-14T03:43:56.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:56.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:43:56.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:56.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:56 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4009832503' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:56.926 INFO:tasks.rados.rados.0.smithi110.stdout:1858: done (6 left) 2024-09-14T03:43:56.927 INFO:tasks.rados.rados.0.smithi110.stdout:1859: done (5 left) 2024-09-14T03:43:56.927 INFO:tasks.rados.rados.0.smithi110.stdout:1860: done (4 left) 2024-09-14T03:43:56.927 INFO:tasks.rados.rados.0.smithi110.stdout:1861: done (3 left) 2024-09-14T03:43:56.927 INFO:tasks.rados.rados.0.smithi110.stdout:1862: done (2 left) 2024-09-14T03:43:56.927 INFO:tasks.rados.rados.0.smithi110.stdout:1863: done (1 left) 2024-09-14T03:43:56.927 INFO:tasks.rados.rados.0.smithi110.stdout:1864: done (0 left) 2024-09-14T03:43:56.928 INFO:tasks.rados.rados.0.smithi110.stdout:1865: snap_create 2024-09-14T03:43:56.959 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:43:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:43:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:43:56.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:56 smithi195 ceph-mon[82388]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:56.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:56 smithi195 ceph-mon[82388]: osdmap e437: 8 total, 8 up, 8 in 2024-09-14T03:43:56.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:56 smithi195 ceph-mon[82388]: pgmap v283: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 92 KiB/s wr, 15 op/s 2024-09-14T03:43:56.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:56 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:56.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:56 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:43:56.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:56 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:43:56.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:56 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4009832503' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:43:57.723 INFO:tasks.rados.rados.0.smithi110.stdout:1865: done (0 left) 2024-09-14T03:43:57.723 INFO:tasks.rados.rados.0.smithi110.stdout:1866: read oid 37 snap -1 2024-09-14T03:43:57.723 INFO:tasks.rados.rados.0.smithi110.stdout:1866: expect (ObjNum 592 snap 181 seq_num 592) 2024-09-14T03:43:57.723 INFO:tasks.rados.rados.0.smithi110.stdout:1867: rollback oid 24 current snap is 183 2024-09-14T03:43:57.723 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 178 2024-09-14T03:43:57.723 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 183 2024-09-14T03:43:57.723 INFO:tasks.rados.rados.0.smithi110.stdout:1868: seq_num 596 ranges {5718016=385024} 2024-09-14T03:43:57.757 INFO:tasks.rados.rados.0.smithi110.stdout:1868: writing smithi11075562-35 from 5718016 to 6103040 tid 1 2024-09-14T03:43:57.758 INFO:tasks.rados.rados.0.smithi110.stdout:1869: rmattr oid 18 current snap is 183 2024-09-14T03:43:57.758 INFO:tasks.rados.rados.0.smithi110.stdout:1867: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:43:57.758 INFO:tasks.rados.rados.0.smithi110.stdout:1867: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:43:57.758 INFO:tasks.rados.rados.0.smithi110.stdout:1867: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:43:57.758 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 912 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:43:57.758 INFO:tasks.rados.rados.0.smithi110.stdout:1866: expect (ObjNum 592 snap 181 seq_num 592) 2024-09-14T03:43:57.913 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:57 smithi195 ceph-mon[82388]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:57.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:57 smithi195 ceph-mon[82388]: Upgrade: Updating osd.4 2024-09-14T03:43:57.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:57 smithi195 ceph-mon[82388]: Deploying daemon osd.4 on smithi195 2024-09-14T03:43:57.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:57 smithi195 ceph-mon[82388]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:57.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:57 smithi195 ceph-mon[82388]: osdmap e438: 8 total, 8 up, 8 in 2024-09-14T03:43:57.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:57 smithi195 ceph-mon[82388]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:57.915 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:57.915 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:58.092 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1062 (ObjNum 595 snap 181 seq_num 595) dirty exists 2024-09-14T03:43:58.092 INFO:tasks.rados.rados.0.smithi110.stdout:1868: finishing write tid 1 to smithi11075562-35 2024-09-14T03:43:58.092 INFO:tasks.rados.rados.0.smithi110.stdout:1868: finishing write tid 2 to smithi11075562-35 2024-09-14T03:43:58.092 INFO:tasks.rados.rados.0.smithi110.stdout:1868: finishing write tid 3 to smithi11075562-35 2024-09-14T03:43:58.093 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1190 (ObjNum 596 snap 183 seq_num 596) dirty exists 2024-09-14T03:43:58.093 INFO:tasks.rados.rados.0.smithi110.stdout:1868: left oid 35 (ObjNum 596 snap 183 seq_num 596) 2024-09-14T03:43:58.093 INFO:tasks.rados.rados.0.smithi110.stdout:1866: done (3 left) 2024-09-14T03:43:58.093 INFO:tasks.rados.rados.0.smithi110.stdout:1867: done (2 left) 2024-09-14T03:43:58.093 INFO:tasks.rados.rados.0.smithi110.stdout:1868: done (1 left) 2024-09-14T03:43:58.093 INFO:tasks.rados.rados.0.smithi110.stdout:1869: done (0 left) 2024-09-14T03:43:58.094 INFO:tasks.rados.rados.0.smithi110.stdout:1870: snap_remove snap 179 2024-09-14T03:43:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[86423]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[86423]: Upgrade: Updating osd.4 2024-09-14T03:43:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[86423]: Deploying daemon osd.4 on smithi195 2024-09-14T03:43:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[86423]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[86423]: osdmap e438: 8 total, 8 up, 8 in 2024-09-14T03:43:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[86423]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[88262]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[88262]: Upgrade: Updating osd.4 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[88262]: Deploying daemon osd.4 on smithi195 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[88262]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[88262]: osdmap e438: 8 total, 8 up, 8 in 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[88262]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:43:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:43:58.708 INFO:tasks.rados.rados.0.smithi110.stdout:1870: done (0 left) 2024-09-14T03:43:58.709 INFO:tasks.rados.rados.0.smithi110.stdout:1871: copy_from oid 27 from oid 35 current snap is 183 2024-09-14T03:43:58.709 INFO:tasks.rados.rados.0.smithi110.stdout:1872: snap_remove snap 176 2024-09-14T03:43:59.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:58 smithi195 ceph-mon[82388]: osdmap e439: 8 total, 8 up, 8 in 2024-09-14T03:43:59.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:58 smithi195 ceph-mon[82388]: pgmap v286: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 504 KiB/s wr, 13 op/s 2024-09-14T03:43:59.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:58 smithi195 ceph-mon[82388]: osdmap e440: 8 total, 8 up, 8 in 2024-09-14T03:43:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:58 smithi110 ceph-mon[88262]: osdmap e439: 8 total, 8 up, 8 in 2024-09-14T03:43:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:58 smithi110 ceph-mon[88262]: pgmap v286: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 504 KiB/s wr, 13 op/s 2024-09-14T03:43:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:58 smithi110 ceph-mon[88262]: osdmap e440: 8 total, 8 up, 8 in 2024-09-14T03:43:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:58 smithi110 ceph-mon[86423]: osdmap e439: 8 total, 8 up, 8 in 2024-09-14T03:43:59.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:58 smithi110 ceph-mon[86423]: pgmap v286: 121 pgs: 121 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 504 KiB/s wr, 13 op/s 2024-09-14T03:43:59.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:58 smithi110 ceph-mon[86423]: osdmap e440: 8 total, 8 up, 8 in 2024-09-14T03:43:59.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:43:59 smithi195 systemd[1]: Stopping Ceph osd.4 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:43:59.718 INFO:tasks.rados.rados.0.smithi110.stdout:1872: done (1 left) 2024-09-14T03:43:59.718 INFO:tasks.rados.rados.0.smithi110.stdout:1873: read oid 34 snap -1 2024-09-14T03:43:59.718 INFO:tasks.rados.rados.0.smithi110.stdout:1873: expect deleted 2024-09-14T03:43:59.718 INFO:tasks.rados.rados.0.smithi110.stdout:1874: delete oid 47 current snap is 183 2024-09-14T03:43:59.718 INFO:tasks.rados.rados.0.smithi110.stdout:1871: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:43:59.719 INFO:tasks.rados.rados.0.smithi110.stdout:1871: finishing copy_from to smithi11075562-27 2024-09-14T03:43:59.719 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1190 (ObjNum 596 snap 183 seq_num 596) dirty exists 2024-09-14T03:43:59.726 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:43:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[35489]: 2024-09-14T03:43:59.538+0000 7f83f3f73640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:43:59.726 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:43:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[35489]: 2024-09-14T03:43:59.538+0000 7f83f3f73640 -1 osd.4 440 *** Got signal Terminated *** 2024-09-14T03:43:59.726 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:43:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[35489]: 2024-09-14T03:43:59.538+0000 7f83f3f73640 -1 osd.4 440 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:44:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:59 smithi110 ceph-mon[88262]: osd.4 marked itself down and dead 2024-09-14T03:44:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:59 smithi110 ceph-mon[88262]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:44:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:43:59 smithi110 ceph-mon[88262]: osdmap e441: 8 total, 7 up, 8 in 2024-09-14T03:44:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:59 smithi110 ceph-mon[86423]: osd.4 marked itself down and dead 2024-09-14T03:44:00.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:59 smithi110 ceph-mon[86423]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:44:00.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:43:59 smithi110 ceph-mon[86423]: osdmap e441: 8 total, 7 up, 8 in 2024-09-14T03:44:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:59 smithi195 ceph-mon[82388]: osd.4 marked itself down and dead 2024-09-14T03:44:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:59 smithi195 ceph-mon[82388]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:44:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:43:59 smithi195 ceph-mon[82388]: osdmap e441: 8 total, 7 up, 8 in 2024-09-14T03:44:00.575 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:00 smithi195 podman[86640]: 2024-09-14 03:44:00.285891797 +0000 UTC m=+0.875300835 container died 43ef4aa248ac2fe641026855395533b3961aa0d60577ef93097019e6d1da223f (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731) 2024-09-14T03:44:00.575 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:00.728 INFO:tasks.rados.rados.0.smithi110.stdout:1871: done (2 left) 2024-09-14T03:44:00.728 INFO:tasks.rados.rados.0.smithi110.stdout:1873: done (1 left) 2024-09-14T03:44:00.728 INFO:tasks.rados.rados.0.smithi110.stdout:1874: done (0 left) 2024-09-14T03:44:00.728 INFO:tasks.rados.rados.0.smithi110.stdout:1875: read oid 32 snap -1 2024-09-14T03:44:00.729 INFO:tasks.rados.rados.0.smithi110.stdout:1875: expect (ObjNum 580 snap 176 seq_num 580) 2024-09-14T03:44:00.729 INFO:tasks.rados.rados.0.smithi110.stdout:1876: snap_remove snap 182 2024-09-14T03:44:00.933 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:00 smithi195 ceph-mon[82388]: pgmap v289: 121 pgs: 8 stale+active+clean, 113 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-14T03:44:00.933 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:00 smithi195 ceph-mon[82388]: osdmap e442: 8 total, 7 up, 8 in 2024-09-14T03:44:00.933 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:00 smithi195 podman[86640]: 2024-09-14 03:44:00.672637693 +0000 UTC m=+1.262046732 container cleanup 43ef4aa248ac2fe641026855395533b3961aa0d60577ef93097019e6d1da223f (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97) 2024-09-14T03:44:00.933 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:00 smithi195 bash[86640]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4 2024-09-14T03:44:01.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:00 smithi110 ceph-mon[88262]: pgmap v289: 121 pgs: 8 stale+active+clean, 113 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-14T03:44:01.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:00 smithi110 ceph-mon[88262]: osdmap e442: 8 total, 7 up, 8 in 2024-09-14T03:44:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:00 smithi110 ceph-mon[86423]: pgmap v289: 121 pgs: 8 stale+active+clean, 113 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-14T03:44:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:00 smithi110 ceph-mon[86423]: osdmap e442: 8 total, 7 up, 8 in 2024-09-14T03:44:01.208 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:00 smithi195 podman[86656]: 2024-09-14 03:44:00.932212802 +0000 UTC m=+0.640939515 container remove 43ef4aa248ac2fe641026855395533b3961aa0d60577ef93097019e6d1da223f (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, GIT_CLEAN=True) 2024-09-14T03:44:01.586 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:01 smithi195 podman[86746]: 2024-09-14 03:44:01.162699262 +0000 UTC m=+0.016021128 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:44:01.587 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:01 smithi195 podman[86746]: 2024-09-14 03:44:01.283362454 +0000 UTC m=+0.136684347 container create d23ccad87402d17d70eac6e4cee47c9fbd40b5da2afe2b2fd11e44d5703a9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True) 2024-09-14T03:44:01.587 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:01 smithi195 podman[86746]: 2024-09-14 03:44:01.502239603 +0000 UTC m=+0.355561452 container init d23ccad87402d17d70eac6e4cee47c9fbd40b5da2afe2b2fd11e44d5703a9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:44:01.587 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:01 smithi195 podman[86746]: 2024-09-14 03:44:01.505544437 +0000 UTC m=+0.358866288 container start d23ccad87402d17d70eac6e4cee47c9fbd40b5da2afe2b2fd11e44d5703a9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T03:44:01.587 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:01 smithi195 podman[86746]: 2024-09-14 03:44:01.585820436 +0000 UTC m=+0.439142289 container attach d23ccad87402d17d70eac6e4cee47c9fbd40b5da2afe2b2fd11e44d5703a9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-14T03:44:01.721 INFO:tasks.rados.rados.0.smithi110.stdout:1876: done (1 left) 2024-09-14T03:44:01.721 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 183 2024-09-14T03:44:01.721 INFO:tasks.rados.rados.0.smithi110.stdout:1877: seq_num 598 ranges {0=417792} 2024-09-14T03:44:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:1877: writing smithi11075562-6 from 0 to 417792 tid 1 2024-09-14T03:44:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 183 2024-09-14T03:44:01.723 INFO:tasks.rados.rados.0.smithi110.stdout:1878: seq_num 599 ranges {2605056=434176} 2024-09-14T03:44:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:1878: writing smithi11075562-8 from 2605056 to 3039232 tid 1 2024-09-14T03:44:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 183 2024-09-14T03:44:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:1879: seq_num 600 ranges {0=450560} 2024-09-14T03:44:01.743 INFO:tasks.rados.rados.0.smithi110.stdout:1879: writing smithi11075562-28 from 0 to 450560 tid 1 2024-09-14T03:44:01.743 INFO:tasks.rados.rados.0.smithi110.stdout:1880: snap_create 2024-09-14T03:44:01.743 INFO:tasks.rados.rados.0.smithi110.stdout:1875: expect (ObjNum 580 snap 176 seq_num 580) 2024-09-14T03:44:01.749 INFO:tasks.rados.rados.0.smithi110.stdout:1877: finishing write tid 1 to smithi11075562-6 2024-09-14T03:44:01.749 INFO:tasks.rados.rados.0.smithi110.stdout:1877: finishing write tid 2 to smithi11075562-6 2024-09-14T03:44:01.749 INFO:tasks.rados.rados.0.smithi110.stdout:1877: finishing write tid 3 to smithi11075562-6 2024-09-14T03:44:01.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1066 (ObjNum 598 snap 183 seq_num 598) dirty exists 2024-09-14T03:44:01.749 INFO:tasks.rados.rados.0.smithi110.stdout:1877: left oid 6 (ObjNum 598 snap 183 seq_num 598) 2024-09-14T03:44:01.749 INFO:tasks.rados.rados.0.smithi110.stdout:1878: finishing write tid 1 to smithi11075562-8 2024-09-14T03:44:01.749 INFO:tasks.rados.rados.0.smithi110.stdout:1878: finishing write tid 2 to smithi11075562-8 2024-09-14T03:44:01.752 INFO:tasks.rados.rados.0.smithi110.stdout:1879: finishing write tid 1 to smithi11075562-28 2024-09-14T03:44:01.752 INFO:tasks.rados.rados.0.smithi110.stdout:1878: finishing write tid 3 to smithi11075562-8 2024-09-14T03:44:01.752 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1163 (ObjNum 599 snap 183 seq_num 599) dirty exists 2024-09-14T03:44:01.752 INFO:tasks.rados.rados.0.smithi110.stdout:1878: left oid 8 (ObjNum 599 snap 183 seq_num 599) 2024-09-14T03:44:01.752 INFO:tasks.rados.rados.0.smithi110.stdout:1879: finishing write tid 2 to smithi11075562-28 2024-09-14T03:44:01.753 INFO:tasks.rados.rados.0.smithi110.stdout:1879: finishing write tid 3 to smithi11075562-28 2024-09-14T03:44:01.753 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 701 (ObjNum 600 snap 183 seq_num 600) dirty exists 2024-09-14T03:44:01.753 INFO:tasks.rados.rados.0.smithi110.stdout:1879: left oid 28 (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:44:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:44:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:44:01] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.51.0" 2024-09-14T03:44:01.959 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:01 smithi195 podman[86746]: 2024-09-14 03:44:01.70072036 +0000 UTC m=+0.554042210 container died d23ccad87402d17d70eac6e4cee47c9fbd40b5da2afe2b2fd11e44d5703a9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T03:44:02.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 podman[86769]: 2024-09-14 03:44:02.289604051 +0000 UTC m=+0.582721465 container remove d23ccad87402d17d70eac6e4cee47c9fbd40b5da2afe2b2fd11e44d5703a9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2) 2024-09-14T03:44:02.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-14T03:44:02.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 systemd[1]: Stopped Ceph osd.4 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:44:02.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.4.service: Consumed 18.582s CPU time. 2024-09-14T03:44:02.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 systemd[1]: Starting Ceph osd.4 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:44:02.725 INFO:tasks.rados.rados.0.smithi110.stdout:1875: done (4 left) 2024-09-14T03:44:02.725 INFO:tasks.rados.rados.0.smithi110.stdout:1877: done (3 left) 2024-09-14T03:44:02.725 INFO:tasks.rados.rados.0.smithi110.stdout:1878: done (2 left) 2024-09-14T03:44:02.725 INFO:tasks.rados.rados.0.smithi110.stdout:1879: done (1 left) 2024-09-14T03:44:02.725 INFO:tasks.rados.rados.0.smithi110.stdout:1880: done (0 left) 2024-09-14T03:44:02.725 INFO:tasks.rados.rados.0.smithi110.stdout:1881: rmattr oid 13 current snap is 184 2024-09-14T03:44:02.725 INFO:tasks.rados.rados.0.smithi110.stdout:1882: read oid 26 snap 177 2024-09-14T03:44:02.725 INFO:tasks.rados.rados.0.smithi110.stdout:1882: expect deleted 2024-09-14T03:44:02.726 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 184 2024-09-14T03:44:02.726 INFO:tasks.rados.rados.0.smithi110.stdout:1883: seq_num 601 ranges {417792=475136} 2024-09-14T03:44:02.730 INFO:tasks.rados.rados.0.smithi110.stdout:1883: writing smithi11075562-6 from 417792 to 892928 tid 1 2024-09-14T03:44:02.730 INFO:tasks.rados.rados.0.smithi110.stdout:1884: rollback oid 24 current snap is 184 2024-09-14T03:44:02.730 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 177 2024-09-14T03:44:02.731 INFO:tasks.rados.rados.0.smithi110.stdout:1882: done (3 left) 2024-09-14T03:44:02.731 INFO:tasks.rados.rados.0.smithi110.stdout:1885: read oid 20 snap -1 2024-09-14T03:44:02.731 INFO:tasks.rados.rados.0.smithi110.stdout:1885: expect (ObjNum 515 snap 160 seq_num 515) 2024-09-14T03:44:02.731 INFO:tasks.rados.rados.0.smithi110.stdout:1886: snap_remove snap 181 2024-09-14T03:44:02.985 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:02 smithi195 ceph-mon[82388]: osdmap e443: 8 total, 7 up, 8 in 2024-09-14T03:44:02.986 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:02 smithi195 ceph-mon[82388]: pgmap v292: 121 pgs: 4 peering, 7 stale+active+clean, 110 active+clean; 211 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T03:44:02.986 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 podman[86868]: 2024-09-14 03:44:02.622842742 +0000 UTC m=+0.015890602 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:44:02.986 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 podman[86868]: 2024-09-14 03:44:02.724130423 +0000 UTC m=+0.117178315 container create e881f9a90f91de09707c47a3a51e16a431e7f8e28008d6441ac67c017e1aa5e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-14T03:44:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:02 smithi110 ceph-mon[88262]: osdmap e443: 8 total, 7 up, 8 in 2024-09-14T03:44:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:02 smithi110 ceph-mon[88262]: pgmap v292: 121 pgs: 4 peering, 7 stale+active+clean, 110 active+clean; 211 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T03:44:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:02 smithi110 ceph-mon[86423]: osdmap e443: 8 total, 7 up, 8 in 2024-09-14T03:44:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:02 smithi110 ceph-mon[86423]: pgmap v292: 121 pgs: 4 peering, 7 stale+active+clean, 110 active+clean; 211 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T03:44:03.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 podman[86868]: 2024-09-14 03:44:02.985135746 +0000 UTC m=+0.378183596 container init e881f9a90f91de09707c47a3a51e16a431e7f8e28008d6441ac67c017e1aa5e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-a278231, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:44:03.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:02 smithi195 podman[86868]: 2024-09-14 03:44:02.988643957 +0000 UTC m=+0.381691809 container start e881f9a90f91de09707c47a3a51e16a431e7f8e28008d6441ac67c017e1aa5e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:44:03.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 podman[86868]: 2024-09-14 03:44:03.076783032 +0000 UTC m=+0.469830883 container attach e881f9a90f91de09707c47a3a51e16a431e7f8e28008d6441ac67c017e1aa5e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-14T03:44:03.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:44:03.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:44:03.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:44:03.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:44:03.709 INFO:tasks.rados.rados.0.smithi110.stdout:1886: done (4 left) 2024-09-14T03:44:03.709 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 184 2024-09-14T03:44:03.710 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1196 (ObjNum 550 snap 167 seq_num 550) dirty exists 2024-09-14T03:44:03.710 INFO:tasks.rados.rados.0.smithi110.stdout:1887: seq_num 602 ranges {6651904=491520} 2024-09-14T03:44:03.749 INFO:tasks.rados.rados.0.smithi110.stdout:1887: writing smithi11075562-25 from 6651904 to 7143424 tid 1 2024-09-14T03:44:03.749 INFO:tasks.rados.rados.0.smithi110.stdout:1881: done (4 left) 2024-09-14T03:44:03.749 INFO:tasks.rados.rados.0.smithi110.stdout:1888: rollback oid 34 current snap is 184 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 183 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1884: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1884: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1884: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 915 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1884: done (4 left) 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1889: rmattr oid 39 current snap is 184 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1889: done (4 left) 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1890: rollback oid 43 current snap is 184 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 43 to 177 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1883: finishing write tid 1 to smithi11075562-6 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1883: finishing write tid 2 to smithi11075562-6 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:1883: finishing write tid 3 to smithi11075562-6 2024-09-14T03:44:03.750 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1069 (ObjNum 601 snap 184 seq_num 601) dirty exists 2024-09-14T03:44:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1883: left oid 6 (ObjNum 601 snap 184 seq_num 601) 2024-09-14T03:44:03.751 INFO:tasks.rados.rados.0.smithi110.stdout:1885: expect (ObjNum 515 snap 160 seq_num 515) 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:1887: finishing write tid 1 to smithi11075562-25 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:1887: finishing write tid 2 to smithi11075562-25 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:1888: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:1890: finishing rollback tid 0 to smithi11075562-43 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:1887: finishing write tid 3 to smithi11075562-25 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1376 (ObjNum 602 snap 184 seq_num 602) dirty exists 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:1887: left oid 25 (ObjNum 602 snap 184 seq_num 602) 2024-09-14T03:44:03.808 INFO:tasks.rados.rados.0.smithi110.stdout:1890: finishing rollback tid 1 to smithi11075562-43 2024-09-14T03:44:03.809 INFO:tasks.rados.rados.0.smithi110.stdout:1883: done (4 left) 2024-09-14T03:44:03.809 INFO:tasks.rados.rados.0.smithi110.stdout:1885: done (3 left) 2024-09-14T03:44:03.809 INFO:tasks.rados.rados.0.smithi110.stdout:1887: done (2 left) 2024-09-14T03:44:03.809 INFO:tasks.rados.rados.0.smithi110.stdout:1888: done (1 left) 2024-09-14T03:44:03.809 INFO:tasks.rados.rados.0.smithi110.stdout:1891: setattr oid 36 current snap is 184 2024-09-14T03:44:03.809 INFO:tasks.rados.rados.0.smithi110.stdout:1890: finishing rollback tid 2 to smithi11075562-43 2024-09-14T03:44:03.809 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1075 (ObjNum 413 snap 134 seq_num 413) dirty exists 2024-09-14T03:44:03.810 INFO:tasks.rados.rados.0.smithi110.stdout:1890: done (1 left) 2024-09-14T03:44:03.810 INFO:tasks.rados.rados.0.smithi110.stdout:1892: read oid 28 snap -1 2024-09-14T03:44:03.810 INFO:tasks.rados.rados.0.smithi110.stdout:1892: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:44:03.810 INFO:tasks.rados.rados.0.smithi110.stdout:1893: rollback oid 45 current snap is 184 2024-09-14T03:44:03.810 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 177 2024-09-14T03:44:03.810 INFO:tasks.rados.rados.0.smithi110.stdout:1894: snap_create 2024-09-14T03:44:03.811 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1378 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-09-14T03:44:03.811 INFO:tasks.rados.rados.0.smithi110.stdout:1892: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:44:03.825 INFO:tasks.rados.rados.0.smithi110.stdout:1893: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:44:03.825 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:44:03.872 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-mon[82388]: osdmap e444: 8 total, 7 up, 8 in 2024-09-14T03:44:03.872 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-mon[82388]: osdmap e445: 8 total, 7 up, 8 in 2024-09-14T03:44:03.872 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T03:44:03.872 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T03:44:03.873 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:44:03.873 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:44:03.873 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:44:04.148 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:44:04.148 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:44:04.149 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:44:04.149 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-14T03:44:04.149 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-14T03:44:04.149 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T03:44:04.149 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T03:44:04.149 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate[86883]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-14T03:44:04.149 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 bash[86868]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-14T03:44:04.149 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:03 smithi195 podman[86868]: 2024-09-14 03:44:03.899103722 +0000 UTC m=+1.292151573 container died e881f9a90f91de09707c47a3a51e16a431e7f8e28008d6441ac67c017e1aa5e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-14T03:44:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:03 smithi110 ceph-mon[88262]: osdmap e444: 8 total, 7 up, 8 in 2024-09-14T03:44:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:03 smithi110 ceph-mon[88262]: osdmap e445: 8 total, 7 up, 8 in 2024-09-14T03:44:04.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:03 smithi110 ceph-mon[86423]: osdmap e444: 8 total, 7 up, 8 in 2024-09-14T03:44:04.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:03 smithi110 ceph-mon[86423]: osdmap e445: 8 total, 7 up, 8 in 2024-09-14T03:44:04.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:04.730 INFO:tasks.rados.rados.0.smithi110.stdout:1891: done (3 left) 2024-09-14T03:44:04.730 INFO:tasks.rados.rados.0.smithi110.stdout:1892: done (2 left) 2024-09-14T03:44:04.730 INFO:tasks.rados.rados.0.smithi110.stdout:1893: done (1 left) 2024-09-14T03:44:04.730 INFO:tasks.rados.rados.0.smithi110.stdout:1894: done (0 left) 2024-09-14T03:44:04.730 INFO:tasks.rados.rados.0.smithi110.stdout:1895: rmattr oid 18 current snap is 185 2024-09-14T03:44:04.730 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 185 2024-09-14T03:44:04.730 INFO:tasks.rados.rados.0.smithi110.stdout:1896: seq_num 603 ranges {10838016=507904} 2024-09-14T03:44:04.790 INFO:tasks.rados.rados.0.smithi110.stdout:1896: writing smithi11075562-14 from 10838016 to 11345920 tid 1 2024-09-14T03:44:04.790 INFO:tasks.rados.rados.0.smithi110.stdout:1897: snap_create 2024-09-14T03:44:04.790 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1071 (ObjNum 595 snap 181 seq_num 595) dirty exists 2024-09-14T03:44:04.795 INFO:tasks.rados.rados.0.smithi110.stdout:1896: finishing write tid 1 to smithi11075562-14 2024-09-14T03:44:04.795 INFO:tasks.rados.rados.0.smithi110.stdout:1896: finishing write tid 2 to smithi11075562-14 2024-09-14T03:44:04.796 INFO:tasks.rados.rados.0.smithi110.stdout:1896: finishing write tid 3 to smithi11075562-14 2024-09-14T03:44:04.797 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1078 (ObjNum 603 snap 185 seq_num 603) dirty exists 2024-09-14T03:44:04.797 INFO:tasks.rados.rados.0.smithi110.stdout:1896: left oid 14 (ObjNum 603 snap 185 seq_num 603) 2024-09-14T03:44:04.862 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:04 smithi195 ceph-mon[82388]: pgmap v295: 121 pgs: 25 active+undersized, 4 peering, 2 stale+active+clean, 9 active+undersized+degraded, 81 active+clean; 212 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 40/282 objects degraded (14.184%) 2024-09-14T03:44:04.862 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:04 smithi195 podman[87007]: 2024-09-14 03:44:04.515039667 +0000 UTC m=+0.605512356 container remove e881f9a90f91de09707c47a3a51e16a431e7f8e28008d6441ac67c017e1aa5e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-14T03:44:04.862 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:04 smithi195 podman[87069]: 2024-09-14 03:44:04.624175755 +0000 UTC m=+0.016201856 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:44:04.862 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:04 smithi195 podman[87069]: 2024-09-14 03:44:04.731573093 +0000 UTC m=+0.123599181 container create 6921b0a30047454dce0ad05a50f5fcc27b1f7841cab6b161f05557566cccd172 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-14T03:44:05.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:04 smithi110 ceph-mon[88262]: pgmap v295: 121 pgs: 25 active+undersized, 4 peering, 2 stale+active+clean, 9 active+undersized+degraded, 81 active+clean; 212 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 40/282 objects degraded (14.184%) 2024-09-14T03:44:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:04 smithi110 ceph-mon[86423]: pgmap v295: 121 pgs: 25 active+undersized, 4 peering, 2 stale+active+clean, 9 active+undersized+degraded, 81 active+clean; 212 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 40/282 objects degraded (14.184%) 2024-09-14T03:44:05.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:04 smithi195 podman[87069]: 2024-09-14 03:44:04.975886061 +0000 UTC m=+0.367912158 container init 6921b0a30047454dce0ad05a50f5fcc27b1f7841cab6b161f05557566cccd172 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-a278231, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:44:05.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:04 smithi195 podman[87069]: 2024-09-14 03:44:04.979430597 +0000 UTC m=+0.371456688 container start 6921b0a30047454dce0ad05a50f5fcc27b1f7841cab6b161f05557566cccd172 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, RELEASE=main-a278231, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:44:05.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:05 smithi195 ceph-osd[87088]: -- 172.21.15.195:0/1934904710 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cc2c013a40 con 0x55cc2bff1c00 2024-09-14T03:44:05.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:05 smithi195 bash[87069]: 6921b0a30047454dce0ad05a50f5fcc27b1f7841cab6b161f05557566cccd172 2024-09-14T03:44:05.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:05 smithi195 systemd[1]: Started Ceph osd.4 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:44:05.732 INFO:tasks.rados.rados.0.smithi110.stdout:1895: done (2 left) 2024-09-14T03:44:05.732 INFO:tasks.rados.rados.0.smithi110.stdout:1896: done (1 left) 2024-09-14T03:44:05.732 INFO:tasks.rados.rados.0.smithi110.stdout:1897: done (0 left) 2024-09-14T03:44:05.732 INFO:tasks.rados.rados.0.smithi110.stdout:1898: copy_from oid 34 from oid 41 current snap is 186 2024-09-14T03:44:05.732 INFO:tasks.rados.rados.0.smithi110.stdout:1899: rollback oid 2 current snap is 186 2024-09-14T03:44:05.732 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 2 to 183 2024-09-14T03:44:05.732 INFO:tasks.rados.rados.0.smithi110.stdout:1900: rollback oid 26 current snap is 186 2024-09-14T03:44:05.732 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 184 2024-09-14T03:44:05.733 INFO:tasks.rados.rados.0.smithi110.stdout:1901: read oid 9 snap -1 2024-09-14T03:44:05.733 INFO:tasks.rados.rados.0.smithi110.stdout:1901: expect (ObjNum 543 snap 166 seq_num 543) 2024-09-14T03:44:05.733 INFO:tasks.rados.rados.0.smithi110.stdout:1902: rollback oid 38 current snap is 186 2024-09-14T03:44:05.733 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 177 2024-09-14T03:44:05.733 INFO:tasks.rados.rados.0.smithi110.stdout:1903: snap_remove snap 178 2024-09-14T03:44:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:05 smithi195 ceph-mon[82388]: Health check failed: Degraded data redundancy: 40/282 objects degraded (14.184%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:05 smithi195 ceph-mon[82388]: osdmap e446: 8 total, 7 up, 8 in 2024-09-14T03:44:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:05 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:05 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:05 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:05.960 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[88262]: Health check failed: Degraded data redundancy: 40/282 objects degraded (14.184%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[88262]: osdmap e446: 8 total, 7 up, 8 in 2024-09-14T03:44:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[86423]: Health check failed: Degraded data redundancy: 40/282 objects degraded (14.184%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:06.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[86423]: osdmap e446: 8 total, 7 up, 8 in 2024-09-14T03:44:06.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:06.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:06.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:05 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:06.741 INFO:tasks.rados.rados.0.smithi110.stdout:1903: done (5 left) 2024-09-14T03:44:06.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 186 2024-09-14T03:44:06.741 INFO:tasks.rados.rados.0.smithi110.stdout:1900: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:44:06.742 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:44:06.742 INFO:tasks.rados.rados.0.smithi110.stdout:1904: seq_num 604 ranges {0=524288} 2024-09-14T03:44:06.743 INFO:tasks.rados.rados.0.smithi110.stdout:1904: writing smithi11075562-21 from 0 to 524288 tid 1 2024-09-14T03:44:06.744 INFO:tasks.rados.rados.0.smithi110.stdout:1900: done (5 left) 2024-09-14T03:44:06.744 INFO:tasks.rados.rados.0.smithi110.stdout:1905: snap_remove snap 185 2024-09-14T03:44:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:06 smithi110 ceph-mon[88262]: osdmap e447: 8 total, 7 up, 8 in 2024-09-14T03:44:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:06 smithi110 ceph-mon[88262]: pgmap v298: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 434 KiB/s wr, 7 op/s; 58/288 objects degraded (20.139%) 2024-09-14T03:44:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:06 smithi110 ceph-mon[86423]: osdmap e447: 8 total, 7 up, 8 in 2024-09-14T03:44:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:06 smithi110 ceph-mon[86423]: pgmap v298: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 434 KiB/s wr, 7 op/s; 58/288 objects degraded (20.139%) 2024-09-14T03:44:07.208 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:44:06.867+0000 7f2bdecbd640 -1 Falling back to public interface 2024-09-14T03:44:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:06 smithi195 ceph-mon[82388]: osdmap e447: 8 total, 7 up, 8 in 2024-09-14T03:44:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:06 smithi195 ceph-mon[82388]: pgmap v298: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 434 KiB/s wr, 7 op/s; 58/288 objects degraded (20.139%) 2024-09-14T03:44:07.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:07.742 INFO:tasks.rados.rados.0.smithi110.stdout:1905: done (5 left) 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:1906: snap_create 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:1902: finishing rollback tid 0 to smithi11075562-38 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:1902: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:1899: finishing rollback tid 0 to smithi11075562-2 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:1899: finishing rollback tid 1 to smithi11075562-2 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:1902: finishing rollback tid 2 to smithi11075562-38 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 606 (ObjNum 519 snap 163 seq_num 519) dirty exists 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:1899: finishing rollback tid 2 to smithi11075562-2 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 704 (ObjNum 455 snap 147 seq_num 455) dirty exists 2024-09-14T03:44:07.743 INFO:tasks.rados.rados.0.smithi110.stdout:1901: expect (ObjNum 543 snap 166 seq_num 543) 2024-09-14T03:44:07.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:07 smithi195 ceph-mon[82388]: osdmap e448: 8 total, 7 up, 8 in 2024-09-14T03:44:08.000 INFO:tasks.rados.rados.0.smithi110.stdout:1898: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:44:08.000 INFO:tasks.rados.rados.0.smithi110.stdout:1898: finishing copy_from to smithi11075562-34 2024-09-14T03:44:08.000 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1054 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:44:08.000 INFO:tasks.rados.rados.0.smithi110.stdout:1904: finishing write tid 1 to smithi11075562-21 2024-09-14T03:44:08.000 INFO:tasks.rados.rados.0.smithi110.stdout:1904: finishing write tid 2 to smithi11075562-21 2024-09-14T03:44:08.001 INFO:tasks.rados.rados.0.smithi110.stdout:1904: finishing write tid 3 to smithi11075562-21 2024-09-14T03:44:08.001 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 1080 (ObjNum 604 snap 186 seq_num 604) dirty exists 2024-09-14T03:44:08.001 INFO:tasks.rados.rados.0.smithi110.stdout:1904: left oid 21 (ObjNum 604 snap 186 seq_num 604) 2024-09-14T03:44:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:07 smithi110 ceph-mon[88262]: osdmap e448: 8 total, 7 up, 8 in 2024-09-14T03:44:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:07 smithi110 ceph-mon[86423]: osdmap e448: 8 total, 7 up, 8 in 2024-09-14T03:44:08.710 INFO:tasks.rados.rados.0.smithi110.stdout:1898: done (5 left) 2024-09-14T03:44:08.710 INFO:tasks.rados.rados.0.smithi110.stdout:1899: done (4 left) 2024-09-14T03:44:08.710 INFO:tasks.rados.rados.0.smithi110.stdout:1901: done (3 left) 2024-09-14T03:44:08.710 INFO:tasks.rados.rados.0.smithi110.stdout:1902: done (2 left) 2024-09-14T03:44:08.710 INFO:tasks.rados.rados.0.smithi110.stdout:1904: done (1 left) 2024-09-14T03:44:08.710 INFO:tasks.rados.rados.0.smithi110.stdout:1906: done (0 left) 2024-09-14T03:44:08.710 INFO:tasks.rados.rados.0.smithi110.stdout:1907: delete oid 42 current snap is 187 2024-09-14T03:44:08.714 INFO:tasks.rados.rados.0.smithi110.stdout:1907: done (0 left) 2024-09-14T03:44:08.714 INFO:tasks.rados.rados.0.smithi110.stdout:1908: copy_from oid 11 from oid 21 current snap is 187 2024-09-14T03:44:08.714 INFO:tasks.rados.rados.0.smithi110.stdout:1909: rollback oid 32 current snap is 187 2024-09-14T03:44:08.714 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 184 2024-09-14T03:44:08.714 INFO:tasks.rados.rados.0.smithi110.stdout:1910: read oid 39 snap -1 2024-09-14T03:44:08.714 INFO:tasks.rados.rados.0.smithi110.stdout:1910: expect (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:44:08.714 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 187 2024-09-14T03:44:08.714 INFO:tasks.rados.rados.0.smithi110.stdout:1911: seq_num 606 ranges {2293760=557056} 2024-09-14T03:44:08.730 INFO:tasks.rados.rados.0.smithi110.stdout:1911: writing smithi11075562-50 from 2293760 to 2850816 tid 1 2024-09-14T03:44:08.730 INFO:tasks.rados.rados.0.smithi110.stdout:1912: snap_remove snap 186 2024-09-14T03:44:08.934 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:08 smithi195 ceph-mon[82388]: osdmap e449: 8 total, 7 up, 8 in 2024-09-14T03:44:08.934 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:08 smithi195 ceph-mon[82388]: pgmap v301: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 588 KiB/s wr, 14 op/s; 65/309 objects degraded (21.036%) 2024-09-14T03:44:08.934 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:08 smithi195 ceph-mon[82388]: osdmap e450: 8 total, 7 up, 8 in 2024-09-14T03:44:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:08 smithi110 ceph-mon[88262]: osdmap e449: 8 total, 7 up, 8 in 2024-09-14T03:44:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:08 smithi110 ceph-mon[88262]: pgmap v301: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 588 KiB/s wr, 14 op/s; 65/309 objects degraded (21.036%) 2024-09-14T03:44:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:08 smithi110 ceph-mon[88262]: osdmap e450: 8 total, 7 up, 8 in 2024-09-14T03:44:09.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:08 smithi110 ceph-mon[86423]: osdmap e449: 8 total, 7 up, 8 in 2024-09-14T03:44:09.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:08 smithi110 ceph-mon[86423]: pgmap v301: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 588 KiB/s wr, 14 op/s; 65/309 objects degraded (21.036%) 2024-09-14T03:44:09.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:08 smithi110 ceph-mon[86423]: osdmap e450: 8 total, 7 up, 8 in 2024-09-14T03:44:09.713 INFO:tasks.rados.rados.0.smithi110.stdout:1912: done (4 left) 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1913: copy_from oid 15 from oid 1 current snap is 187 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1909: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1909: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1909: finishing rollback tid 2 to smithi11075562-32 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1166 (ObjNum 580 snap 176 seq_num 580) dirty exists 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1908: finishing copy_from racing read to smithi11075562-11 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1911: finishing write tid 1 to smithi11075562-50 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1908: finishing copy_from to smithi11075562-11 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1080 (ObjNum 604 snap 186 seq_num 604) dirty exists 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1911: finishing write tid 2 to smithi11075562-50 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1911: finishing write tid 3 to smithi11075562-50 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 937 (ObjNum 606 snap 187 seq_num 606) dirty exists 2024-09-14T03:44:09.714 INFO:tasks.rados.rados.0.smithi110.stdout:1911: left oid 50 (ObjNum 606 snap 187 seq_num 606) 2024-09-14T03:44:09.715 INFO:tasks.rados.rados.0.smithi110.stdout:1910: expect (ObjNum 389 snap 129 seq_num 389) 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1913: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1913: finishing copy_from to smithi11075562-15 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1187 (ObjNum 572 snap 174 seq_num 572) dirty exists 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1908: done (4 left) 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1909: done (3 left) 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1910: done (2 left) 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1911: done (1 left) 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1913: done (0 left) 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1914: rmattr oid 10 current snap is 187 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1914: done (0 left) 2024-09-14T03:44:10.096 INFO:tasks.rados.rados.0.smithi110.stdout:1915: setattr oid 27 current snap is 187 2024-09-14T03:44:10.097 INFO:tasks.rados.rados.0.smithi110.stdout:1916: snap_remove snap 184 2024-09-14T03:44:10.536 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:10.716 INFO:tasks.rados.rados.0.smithi110.stdout:1916: done (1 left) 2024-09-14T03:44:10.716 INFO:tasks.rados.rados.0.smithi110.stdout:1917: setattr oid 33 current snap is 187 2024-09-14T03:44:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1191 (ObjNum 596 snap 183 seq_num 596) dirty exists 2024-09-14T03:44:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1915: done (1 left) 2024-09-14T03:44:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1918: copy_from oid 38 from oid 1 current snap is 187 2024-09-14T03:44:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:1919: rollback oid 15 current snap is 187 2024-09-14T03:44:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 183 2024-09-14T03:44:10.720 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 187 2024-09-14T03:44:10.721 INFO:tasks.rados.rados.0.smithi110.stdout:1920: seq_num 607 ranges {11862016=573440} 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:1920: writing smithi11075562-22 from 11862016 to 12435456 tid 1 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:1921: rollback oid 10 current snap is 187 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 177 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1043 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:1919: finishing rollback tid 0 to smithi11075562-15 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:1919: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:1919: finishing rollback tid 2 to smithi11075562-15 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1190 (ObjNum 575 snap 175 seq_num 575) dirty exists 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:1918: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:44:10.788 INFO:tasks.rados.rados.0.smithi110.stdout:1917: done (4 left) 2024-09-14T03:44:10.789 INFO:tasks.rados.rados.0.smithi110.stdout:1919: done (3 left) 2024-09-14T03:44:10.789 INFO:tasks.rados.rados.0.smithi110.stdout:1922: snap_create 2024-09-14T03:44:10.789 INFO:tasks.rados.rados.0.smithi110.stdout:1918: finishing copy_from to smithi11075562-38 2024-09-14T03:44:10.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1187 (ObjNum 572 snap 174 seq_num 572) dirty exists 2024-09-14T03:44:10.791 INFO:tasks.rados.rados.0.smithi110.stdout:1920: finishing write tid 1 to smithi11075562-22 2024-09-14T03:44:10.791 INFO:tasks.rados.rados.0.smithi110.stdout:1920: finishing write tid 2 to smithi11075562-22 2024-09-14T03:44:10.791 INFO:tasks.rados.rados.0.smithi110.stdout:1921: finishing rollback tid 0 to smithi11075562-10 2024-09-14T03:44:10.792 INFO:tasks.rados.rados.0.smithi110.stdout:1921: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:44:10.792 INFO:tasks.rados.rados.0.smithi110.stdout:1921: finishing rollback tid 2 to smithi11075562-10 2024-09-14T03:44:10.792 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 918 (ObjNum 500 snap 156 seq_num 500) dirty exists 2024-09-14T03:44:10.792 INFO:tasks.rados.rados.0.smithi110.stdout:1920: finishing write tid 3 to smithi11075562-22 2024-09-14T03:44:10.792 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1192 (ObjNum 607 snap 187 seq_num 607) dirty exists 2024-09-14T03:44:10.792 INFO:tasks.rados.rados.0.smithi110.stdout:1920: left oid 22 (ObjNum 607 snap 187 seq_num 607) 2024-09-14T03:44:10.865 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:10 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.865 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:10 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.866 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:10 smithi195 ceph-mon[82388]: osdmap e451: 8 total, 7 up, 8 in 2024-09-14T03:44:10.866 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:10 smithi195 ceph-mon[82388]: pgmap v304: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 372 KiB/s wr, 11 op/s; 66/309 objects degraded (21.359%) 2024-09-14T03:44:10.866 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:10 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.866 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:10 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[86423]: osdmap e451: 8 total, 7 up, 8 in 2024-09-14T03:44:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[86423]: pgmap v304: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 372 KiB/s wr, 11 op/s; 66/309 objects degraded (21.359%) 2024-09-14T03:44:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[88262]: osdmap e451: 8 total, 7 up, 8 in 2024-09-14T03:44:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[88262]: pgmap v304: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 372 KiB/s wr, 11 op/s; 66/309 objects degraded (21.359%) 2024-09-14T03:44:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:11.209 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:44:11.056+0000 7f2bdecbd640 -1 osd.4 440 log_to_monitors true 2024-09-14T03:44:11.718 INFO:tasks.rados.rados.0.smithi110.stdout:1918: done (3 left) 2024-09-14T03:44:11.718 INFO:tasks.rados.rados.0.smithi110.stdout:1920: done (2 left) 2024-09-14T03:44:11.718 INFO:tasks.rados.rados.0.smithi110.stdout:1921: done (1 left) 2024-09-14T03:44:11.718 INFO:tasks.rados.rados.0.smithi110.stdout:1922: done (0 left) 2024-09-14T03:44:11.718 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 188 2024-09-14T03:44:11.718 INFO:tasks.rados.rados.0.smithi110.stdout:1923: seq_num 608 ranges {3039232=589824} 2024-09-14T03:44:11.738 INFO:tasks.rados.rados.0.smithi110.stdout:1923: writing smithi11075562-8 from 3039232 to 3629056 tid 1 2024-09-14T03:44:11.738 INFO:tasks.rados.rados.0.smithi110.stdout:1924: delete oid 12 current snap is 188 2024-09-14T03:44:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:1924: done (1 left) 2024-09-14T03:44:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 188 2024-09-14T03:44:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:1925: seq_num 610 ranges {6103040=622592} 2024-09-14T03:44:11.778 INFO:tasks.rados.rados.0.smithi110.stdout:1925: writing smithi11075562-35 from 6103040 to 6725632 tid 1 2024-09-14T03:44:11.778 INFO:tasks.rados.rados.0.smithi110.stdout:1926: delete oid 4 current snap is 188 2024-09-14T03:44:11.778 INFO:tasks.rados.rados.0.smithi110.stdout:1923: finishing write tid 1 to smithi11075562-8 2024-09-14T03:44:11.778 INFO:tasks.rados.rados.0.smithi110.stdout:1923: finishing write tid 2 to smithi11075562-8 2024-09-14T03:44:11.778 INFO:tasks.rados.rados.0.smithi110.stdout:1923: finishing write tid 3 to smithi11075562-8 2024-09-14T03:44:11.778 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1194 (ObjNum 608 snap 188 seq_num 608) dirty exists 2024-09-14T03:44:11.778 INFO:tasks.rados.rados.0.smithi110.stdout:1923: left oid 8 (ObjNum 608 snap 188 seq_num 608) 2024-09-14T03:44:11.780 INFO:tasks.rados.rados.0.smithi110.stdout:1923: done (2 left) 2024-09-14T03:44:11.780 INFO:tasks.rados.rados.0.smithi110.stdout:1926: done (1 left) 2024-09-14T03:44:11.780 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 188 2024-09-14T03:44:11.780 INFO:tasks.rados.rados.0.smithi110.stdout:1927: seq_num 612 ranges {0=655360} 2024-09-14T03:44:11.784 INFO:tasks.rados.rados.0.smithi110.stdout:1927: writing smithi11075562-47 from 0 to 655360 tid 1 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1928: rmattr oid 23 current snap is 188 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1925: finishing write tid 1 to smithi11075562-35 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1925: finishing write tid 2 to smithi11075562-35 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1925: finishing write tid 3 to smithi11075562-35 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1201 (ObjNum 610 snap 188 seq_num 610) dirty exists 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1925: left oid 35 (ObjNum 610 snap 188 seq_num 610) 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1925: done (2 left) 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1929: copy_from oid 31 from oid 2 current snap is 188 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1930: copy_from oid 7 from oid 46 current snap is 188 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 188 2024-09-14T03:44:11.785 INFO:tasks.rados.rados.0.smithi110.stdout:1931: seq_num 613 ranges {4038656=671744} 2024-09-14T03:44:11.811 INFO:tasks.rados.rados.0.smithi110.stdout:1931: writing smithi11075562-43 from 4038656 to 4710400 tid 1 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:1932: delete oid 48 current snap is 188 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:1927: finishing write tid 1 to smithi11075562-47 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:1927: finishing write tid 2 to smithi11075562-47 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 919 (ObjNum 541 snap 166 seq_num 541) dirty exists 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:1927: finishing write tid 3 to smithi11075562-47 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1383 (ObjNum 612 snap 188 seq_num 612) dirty exists 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:1927: left oid 47 (ObjNum 612 snap 188 seq_num 612) 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:1930: finishing copy_from racing read to smithi11075562-7 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:1930: finishing copy_from to smithi11075562-7 2024-09-14T03:44:11.812 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1039 (ObjNum 584 snap 180 seq_num 584) dirty exists 2024-09-14T03:44:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:1929: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:44:11.816 INFO:tasks.rados.rados.0.smithi110.stdout:1927: done (5 left) 2024-09-14T03:44:11.816 INFO:tasks.rados.rados.0.smithi110.stdout:1928: done (4 left) 2024-09-14T03:44:11.816 INFO:tasks.rados.rados.0.smithi110.stdout:1930: done (3 left) 2024-09-14T03:44:11.816 INFO:tasks.rados.rados.0.smithi110.stdout:1932: done (2 left) 2024-09-14T03:44:11.816 INFO:tasks.rados.rados.0.smithi110.stdout:1933: read oid 16 snap -1 2024-09-14T03:44:11.816 INFO:tasks.rados.rados.0.smithi110.stdout:1933: expect (ObjNum 540 snap 166 seq_num 540) 2024-09-14T03:44:11.816 INFO:tasks.rados.rados.0.smithi110.stdout:1934: delete oid 30 current snap is 188 2024-09-14T03:44:11.817 INFO:tasks.rados.rados.0.smithi110.stdout:1931: finishing write tid 1 to smithi11075562-43 2024-09-14T03:44:11.817 INFO:tasks.rados.rados.0.smithi110.stdout:1931: finishing write tid 2 to smithi11075562-43 2024-09-14T03:44:11.817 INFO:tasks.rados.rados.0.smithi110.stdout:1931: finishing write tid 3 to smithi11075562-43 2024-09-14T03:44:11.817 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1083 (ObjNum 613 snap 188 seq_num 613) dirty exists 2024-09-14T03:44:11.817 INFO:tasks.rados.rados.0.smithi110.stdout:1931: left oid 43 (ObjNum 613 snap 188 seq_num 613) 2024-09-14T03:44:11.818 INFO:tasks.rados.rados.0.smithi110.stdout:1931: done (3 left) 2024-09-14T03:44:11.818 INFO:tasks.rados.rados.0.smithi110.stdout:1934: done (2 left) 2024-09-14T03:44:11.818 INFO:tasks.rados.rados.0.smithi110.stdout:1935: snap_create 2024-09-14T03:44:11.821 INFO:tasks.rados.rados.0.smithi110.stdout:1929: finishing copy_from to smithi11075562-31 2024-09-14T03:44:11.821 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 704 (ObjNum 455 snap 147 seq_num 455) dirty exists 2024-09-14T03:44:11.834 INFO:tasks.rados.rados.0.smithi110.stdout:1933: expect (ObjNum 540 snap 166 seq_num 540) 2024-09-14T03:44:11.868 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:44:11.744+0000 7f2bd6a67640 -1 osd.4 440 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:44:11.868 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:11 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 66/309 objects degraded (21.359%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:11.869 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:11 smithi195 ceph-mon[82388]: osdmap e452: 8 total, 7 up, 8 in 2024-09-14T03:44:11.869 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:11 smithi195 ceph-mon[82388]: from='osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:44:11.869 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:11 smithi195 ceph-mon[82388]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:44:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 66/309 objects degraded (21.359%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-mon[86423]: osdmap e452: 8 total, 7 up, 8 in 2024-09-14T03:44:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-mon[86423]: from='osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:44:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-mon[86423]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:44:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 66/309 objects degraded (21.359%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:11.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-mon[88262]: osdmap e452: 8 total, 7 up, 8 in 2024-09-14T03:44:11.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-mon[88262]: from='osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:44:11.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-mon[88262]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:44:11.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:44:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:44:11] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.51.0" 2024-09-14T03:44:12.723 INFO:tasks.rados.rados.0.smithi110.stdout:1929: done (2 left) 2024-09-14T03:44:12.724 INFO:tasks.rados.rados.0.smithi110.stdout:1933: done (1 left) 2024-09-14T03:44:12.724 INFO:tasks.rados.rados.0.smithi110.stdout:1935: done (0 left) 2024-09-14T03:44:12.724 INFO:tasks.rados.rados.0.smithi110.stdout:1936: delete oid 9 current snap is 189 2024-09-14T03:44:12.741 INFO:tasks.rados.rados.0.smithi110.stdout:1936: done (0 left) 2024-09-14T03:44:12.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 189 2024-09-14T03:44:12.741 INFO:tasks.rados.rados.0.smithi110.stdout:1937: seq_num 617 ranges {6807552=737280} 2024-09-14T03:44:12.782 INFO:tasks.rados.rados.0.smithi110.stdout:1937: writing smithi11075562-39 from 6807552 to 7544832 tid 1 2024-09-14T03:44:12.783 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 189 2024-09-14T03:44:12.783 INFO:tasks.rados.rados.0.smithi110.stdout:1938: seq_num 618 ranges {655360=753664} 2024-09-14T03:44:12.789 INFO:tasks.rados.rados.0.smithi110.stdout:1938: writing smithi11075562-47 from 655360 to 1409024 tid 1 2024-09-14T03:44:12.790 INFO:tasks.rados.rados.0.smithi110.stdout:1939: read oid 32 snap -1 2024-09-14T03:44:12.790 INFO:tasks.rados.rados.0.smithi110.stdout:1939: expect (ObjNum 580 snap 176 seq_num 580) 2024-09-14T03:44:12.790 INFO:tasks.rados.rados.0.smithi110.stdout:1940: snap_create 2024-09-14T03:44:13.186 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:12 smithi195 ceph-mon[82388]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:44:13.186 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:12 smithi195 ceph-mon[82388]: osdmap e453: 8 total, 7 up, 8 in 2024-09-14T03:44:13.186 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:12 smithi195 ceph-mon[82388]: from='osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:13.186 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:12 smithi195 ceph-mon[82388]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:13.186 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:12 smithi195 ceph-mon[82388]: pgmap v307: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 3.0 MiB/s wr, 2 op/s; 68/321 objects degraded (21.184%) 2024-09-14T03:44:13.186 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:12 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:13.186 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[88262]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:44:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[88262]: osdmap e453: 8 total, 7 up, 8 in 2024-09-14T03:44:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[88262]: from='osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[88262]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[88262]: pgmap v307: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 3.0 MiB/s wr, 2 op/s; 68/321 objects degraded (21.184%) 2024-09-14T03:44:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[86423]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:44:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[86423]: osdmap e453: 8 total, 7 up, 8 in 2024-09-14T03:44:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[86423]: from='osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[86423]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[86423]: pgmap v307: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 3.0 MiB/s wr, 2 op/s; 68/321 objects degraded (21.184%) 2024-09-14T03:44:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:13.721 INFO:tasks.rados.rados.0.smithi110.stdout:1940: done (3 left) 2024-09-14T03:44:13.721 INFO:tasks.rados.rados.0.smithi110.stdout:1941: copy_from oid 8 from oid 45 current snap is 190 2024-09-14T03:44:13.721 INFO:tasks.rados.rados.0.smithi110.stdout:1942: delete oid 23 current snap is 190 2024-09-14T03:44:13.742 INFO:tasks.rados.rados.0.smithi110.stdout:1942: done (4 left) 2024-09-14T03:44:13.742 INFO:tasks.rados.rados.0.smithi110.stdout:1943: snap_remove snap 187 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122] boot 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:13 smithi195 ceph-mon[82388]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T03:44:14.196 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:44:13.763+0000 7f940e88b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:44:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:44:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122] boot 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[88262]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:44:14.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: osd.4 [v2:172.21.15.195:6800/2754374122,v1:172.21.15.195:6801/2754374122] boot 2024-09-14T03:44:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T03:44:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:44:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:44:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:13 smithi110 ceph-mon[86423]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T03:44:14.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:14.736 INFO:tasks.rados.rados.0.smithi110.stdout:1943: done (4 left) 2024-09-14T03:44:14.736 INFO:tasks.rados.rados.0.smithi110.stdout:1944: setattr oid 34 current snap is 190 2024-09-14T03:44:14.736 INFO:tasks.rados.rados.0.smithi110.stdout:1937: finishing write tid 1 to smithi11075562-39 2024-09-14T03:44:14.737 INFO:tasks.rados.rados.0.smithi110.stdout:1937: finishing write tid 2 to smithi11075562-39 2024-09-14T03:44:14.737 INFO:tasks.rados.rados.0.smithi110.stdout:1937: finishing write tid 3 to smithi11075562-39 2024-09-14T03:44:14.737 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1086 (ObjNum 617 snap 189 seq_num 617) dirty exists 2024-09-14T03:44:14.737 INFO:tasks.rados.rados.0.smithi110.stdout:1937: left oid 39 (ObjNum 617 snap 189 seq_num 617) 2024-09-14T03:44:14.737 INFO:tasks.rados.rados.0.smithi110.stdout:1939: expect (ObjNum 580 snap 176 seq_num 580) 2024-09-14T03:44:14.741 INFO:tasks.rados.rados.0.smithi110.stdout:1938: finishing write tid 1 to smithi11075562-47 2024-09-14T03:44:14.741 INFO:tasks.rados.rados.0.smithi110.stdout:1938: finishing write tid 2 to smithi11075562-47 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1938: finishing write tid 3 to smithi11075562-47 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1386 (ObjNum 618 snap 189 seq_num 618) dirty exists 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1938: left oid 47 (ObjNum 618 snap 189 seq_num 618) 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1941: finishing copy_from to smithi11075562-8 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1941: got expected ENOENT (src dne) 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1941: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1937: done (4 left) 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1938: done (3 left) 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1939: done (2 left) 2024-09-14T03:44:14.742 INFO:tasks.rados.rados.0.smithi110.stdout:1941: done (1 left) 2024-09-14T03:44:14.743 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 190 2024-09-14T03:44:14.743 INFO:tasks.rados.rados.0.smithi110.stdout:1945: seq_num 620 ranges {188416=794624} 2024-09-14T03:44:14.748 INFO:tasks.rados.rados.0.smithi110.stdout:1945: writing smithi11075562-46 from 188416 to 983040 tid 1 2024-09-14T03:44:14.748 INFO:tasks.rados.rados.0.smithi110.stdout:1946: delete oid 23 current snap is 190 2024-09-14T03:44:14.748 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1152 (ObjNum 426 snap 136 seq_num 426) dirty exists 2024-09-14T03:44:14.750 INFO:tasks.rados.rados.0.smithi110.stdout:1944: done (2 left) 2024-09-14T03:44:14.750 INFO:tasks.rados.rados.0.smithi110.stdout:1946: done (1 left) 2024-09-14T03:44:14.750 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 190 2024-09-14T03:44:14.750 INFO:tasks.rados.rados.0.smithi110.stdout:1947: seq_num 622 ranges {0=425984,425984=401408} 2024-09-14T03:44:14.752 INFO:tasks.rados.rados.0.smithi110.stdout:1947: writing smithi11075562-26 from 0 to 425984 tid 1 2024-09-14T03:44:14.755 INFO:tasks.rados.rados.0.smithi110.stdout:1947: writing smithi11075562-26 from 425984 to 827392 tid 2 2024-09-14T03:44:14.755 INFO:tasks.rados.rados.0.smithi110.stdout:1948: snap_remove snap 177 2024-09-14T03:44:14.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:14.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-mon[82388]: pgmap v310: 121 pgs: 29 peering, 10 active+undersized, 7 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.1 MiB/s wr, 11 op/s; 38/339 objects degraded (11.209%) 2024-09-14T03:44:14.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:14.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:14.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:14.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:44:14.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:44:14.972 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:14 smithi195 ceph-mon[82388]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:44:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[86423]: pgmap v310: 121 pgs: 29 peering, 10 active+undersized, 7 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.1 MiB/s wr, 11 op/s; 38/339 objects degraded (11.209%) 2024-09-14T03:44:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[86423]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[88262]: pgmap v310: 121 pgs: 29 peering, 10 active+undersized, 7 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.1 MiB/s wr, 11 op/s; 38/339 objects degraded (11.209%) 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:44:15.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:14 smithi110 ceph-mon[88262]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:44:15.741 INFO:tasks.rados.rados.0.smithi110.stdout:1945: finishing write tid 1 to smithi11075562-46 2024-09-14T03:44:15.741 INFO:tasks.rados.rados.0.smithi110.stdout:1945: finishing write tid 2 to smithi11075562-46 2024-09-14T03:44:15.741 INFO:tasks.rados.rados.0.smithi110.stdout:1945: finishing write tid 3 to smithi11075562-46 2024-09-14T03:44:15.741 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1046 (ObjNum 620 snap 190 seq_num 620) dirty exists 2024-09-14T03:44:15.741 INFO:tasks.rados.rados.0.smithi110.stdout:1945: left oid 46 (ObjNum 620 snap 190 seq_num 620) 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1947: finishing write tid 1 to smithi11075562-26 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1947: finishing write tid 2 to smithi11075562-26 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1947: finishing write tid 3 to smithi11075562-26 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1947: finishing write tid 4 to smithi11075562-26 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 773 (ObjNum 622 snap 190 seq_num 622) dirty exists 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1947: left oid 26 (ObjNum 622 snap 190 seq_num 622) 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1945: done (2 left) 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1947: done (1 left) 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1948: done (0 left) 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 190 2024-09-14T03:44:15.742 INFO:tasks.rados.rados.0.smithi110.stdout:1949: seq_num 623 ranges {6103040=442368,6545408=401408} 2024-09-14T03:44:15.777 INFO:tasks.rados.rados.0.smithi110.stdout:1949: writing smithi11075562-27 from 6103040 to 6545408 tid 1 2024-09-14T03:44:15.779 INFO:tasks.rados.rados.0.smithi110.stdout:1949: writing smithi11075562-27 from 6545408 to 6946816 tid 2 2024-09-14T03:44:15.779 INFO:tasks.rados.rados.0.smithi110.stdout:1950: snap_create 2024-09-14T03:44:15.781 INFO:tasks.rados.rados.0.smithi110.stdout:1949: finishing write tid 1 to smithi11075562-27 2024-09-14T03:44:15.781 INFO:tasks.rados.rados.0.smithi110.stdout:1949: finishing write tid 2 to smithi11075562-27 2024-09-14T03:44:15.781 INFO:tasks.rados.rados.0.smithi110.stdout:1949: finishing write tid 3 to smithi11075562-27 2024-09-14T03:44:15.782 INFO:tasks.rados.rados.0.smithi110.stdout:1949: finishing write tid 4 to smithi11075562-27 2024-09-14T03:44:15.782 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1194 (ObjNum 623 snap 190 seq_num 623) dirty exists 2024-09-14T03:44:15.783 INFO:tasks.rados.rados.0.smithi110.stdout:1949: left oid 27 (ObjNum 623 snap 190 seq_num 623) 2024-09-14T03:44:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:15 smithi110 ceph-mon[88262]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T03:44:16.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:15 smithi110 ceph-mon[86423]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T03:44:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:15 smithi195 ceph-mon[82388]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T03:44:16.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:16.210 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:16.745 INFO:tasks.rados.rados.0.smithi110.stdout:1949: done (1 left) 2024-09-14T03:44:16.745 INFO:tasks.rados.rados.0.smithi110.stdout:1950: done (0 left) 2024-09-14T03:44:16.745 INFO:tasks.rados.rados.0.smithi110.stdout:1951: rollback oid 36 current snap is 191 2024-09-14T03:44:16.745 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 36 to 189 2024-09-14T03:44:16.745 INFO:tasks.rados.rados.0.smithi110.stdout:1952: snap_create 2024-09-14T03:44:16.752 INFO:tasks.rados.rados.0.smithi110.stdout:1951: finishing rollback tid 0 to smithi11075562-36 2024-09-14T03:44:16.752 INFO:tasks.rados.rados.0.smithi110.stdout:1951: finishing rollback tid 1 to smithi11075562-36 2024-09-14T03:44:16.752 INFO:tasks.rados.rados.0.smithi110.stdout:1951: finishing rollback tid 2 to smithi11075562-36 2024-09-14T03:44:16.752 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1414 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-09-14T03:44:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:16 smithi110 ceph-mon[88262]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T03:44:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:16 smithi110 ceph-mon[88262]: pgmap v313: 121 pgs: 29 peering, 5 active+undersized, 3 active+undersized+degraded, 84 active+clean; 239 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 3.1 MiB/s wr, 27 op/s; 18/366 objects degraded (4.918%); 14 MiB/s, 3 objects/s recovering 2024-09-14T03:44:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:16 smithi110 ceph-mon[86423]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T03:44:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:16 smithi110 ceph-mon[86423]: pgmap v313: 121 pgs: 29 peering, 5 active+undersized, 3 active+undersized+degraded, 84 active+clean; 239 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 3.1 MiB/s wr, 27 op/s; 18/366 objects degraded (4.918%); 14 MiB/s, 3 objects/s recovering 2024-09-14T03:44:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:16 smithi195 ceph-mon[82388]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T03:44:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:16 smithi195 ceph-mon[82388]: pgmap v313: 121 pgs: 29 peering, 5 active+undersized, 3 active+undersized+degraded, 84 active+clean; 239 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 3.1 MiB/s wr, 27 op/s; 18/366 objects degraded (4.918%); 14 MiB/s, 3 objects/s recovering 2024-09-14T03:44:17.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:17.747 INFO:tasks.rados.rados.0.smithi110.stdout:1951: done (1 left) 2024-09-14T03:44:17.747 INFO:tasks.rados.rados.0.smithi110.stdout:1952: done (0 left) 2024-09-14T03:44:17.747 INFO:tasks.rados.rados.0.smithi110.stdout:1953: read oid 4 snap 191 2024-09-14T03:44:17.747 INFO:tasks.rados.rados.0.smithi110.stdout:1953: expect deleted 2024-09-14T03:44:17.747 INFO:tasks.rados.rados.0.smithi110.stdout:1954: snap_remove snap 188 2024-09-14T03:44:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:17 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 18/366 objects degraded (4.918%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:17 smithi110 ceph-mon[88262]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T03:44:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:17 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 18/366 objects degraded (4.918%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:17 smithi110 ceph-mon[86423]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T03:44:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:17 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 18/366 objects degraded (4.918%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:17 smithi195 ceph-mon[82388]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T03:44:18.759 INFO:tasks.rados.rados.0.smithi110.stdout:1953: done (1 left) 2024-09-14T03:44:18.759 INFO:tasks.rados.rados.0.smithi110.stdout:1954: done (0 left) 2024-09-14T03:44:18.759 INFO:tasks.rados.rados.0.smithi110.stdout:1955: setattr oid 3 current snap is 192 2024-09-14T03:44:18.764 INFO:tasks.rados.rados.0.smithi110.stdout:1956: snap_remove snap 189 2024-09-14T03:44:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:18 smithi110 ceph-mon[88262]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T03:44:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:18 smithi110 ceph-mon[88262]: pgmap v316: 121 pgs: 121 active+clean; 234 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 2.9 MiB/s wr, 23 op/s; 40 MiB/s, 10 objects/s recovering 2024-09-14T03:44:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:18 smithi110 ceph-mon[86423]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T03:44:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:18 smithi110 ceph-mon[86423]: pgmap v316: 121 pgs: 121 active+clean; 234 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 2.9 MiB/s wr, 23 op/s; 40 MiB/s, 10 objects/s recovering 2024-09-14T03:44:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:18 smithi195 ceph-mon[82388]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T03:44:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:18 smithi195 ceph-mon[82388]: pgmap v316: 121 pgs: 121 active+clean; 234 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 2.9 MiB/s wr, 23 op/s; 40 MiB/s, 10 objects/s recovering 2024-09-14T03:44:19.763 INFO:tasks.rados.rados.0.smithi110.stdout:1956: done (1 left) 2024-09-14T03:44:19.763 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 192 2024-09-14T03:44:19.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 713 (ObjNum 561 snap 171 seq_num 561) dirty exists 2024-09-14T03:44:19.763 INFO:tasks.rados.rados.0.smithi110.stdout:1957: seq_num 624 ranges {0=458752,458752=401408} 2024-09-14T03:44:19.765 INFO:tasks.rados.rados.0.smithi110.stdout:1957: writing smithi11075562-42 from 0 to 458752 tid 1 2024-09-14T03:44:19.768 INFO:tasks.rados.rados.0.smithi110.stdout:1957: writing smithi11075562-42 from 458752 to 860160 tid 2 2024-09-14T03:44:19.768 INFO:tasks.rados.rados.0.smithi110.stdout:1955: done (1 left) 2024-09-14T03:44:19.768 INFO:tasks.rados.rados.0.smithi110.stdout:1958: copy_from oid 29 from oid 48 current snap is 192 2024-09-14T03:44:19.768 INFO:tasks.rados.rados.0.smithi110.stdout:1959: snap_remove snap 183 2024-09-14T03:44:20.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:19 smithi110 ceph-mon[86423]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/366 objects degraded (4.918%), 3 pgs degraded) 2024-09-14T03:44:20.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:19 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:44:20.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:19 smithi110 ceph-mon[86423]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T03:44:20.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:19 smithi110 ceph-mon[88262]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/366 objects degraded (4.918%), 3 pgs degraded) 2024-09-14T03:44:20.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:19 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:44:20.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:19 smithi110 ceph-mon[88262]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T03:44:20.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:19 smithi195 ceph-mon[82388]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/366 objects degraded (4.918%), 3 pgs degraded) 2024-09-14T03:44:20.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:19 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:44:20.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:19 smithi195 ceph-mon[82388]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T03:44:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:20.768 INFO:tasks.rados.rados.0.smithi110.stdout:1959: done (2 left) 2024-09-14T03:44:20.768 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 192 2024-09-14T03:44:20.768 INFO:tasks.rados.rados.0.smithi110.stdout:1958: finishing copy_from to smithi11075562-29 2024-09-14T03:44:20.768 INFO:tasks.rados.rados.0.smithi110.stdout:1958: got expected ENOENT (src dne) 2024-09-14T03:44:20.768 INFO:tasks.rados.rados.0.smithi110.stdout:1960: seq_num 625 ranges {0=475136,475136=401408} 2024-09-14T03:44:20.770 INFO:tasks.rados.rados.0.smithi110.stdout:1960: writing smithi11075562-49 from 0 to 475136 tid 1 2024-09-14T03:44:20.774 INFO:tasks.rados.rados.0.smithi110.stdout:1960: writing smithi11075562-49 from 475136 to 876544 tid 2 2024-09-14T03:44:20.774 INFO:tasks.rados.rados.0.smithi110.stdout:1961: read oid 30 snap -1 2024-09-14T03:44:20.774 INFO:tasks.rados.rados.0.smithi110.stdout:1961: expect deleted 2024-09-14T03:44:20.774 INFO:tasks.rados.rados.0.smithi110.stdout:1962: snap_create 2024-09-14T03:44:20.774 INFO:tasks.rados.rados.0.smithi110.stdout:1958: finishing copy_from racing read to smithi11075562-29 2024-09-14T03:44:20.774 INFO:tasks.rados.rados.0.smithi110.stdout:1957: finishing write tid 1 to smithi11075562-42 2024-09-14T03:44:20.774 INFO:tasks.rados.rados.0.smithi110.stdout:1957: finishing write tid 2 to smithi11075562-42 2024-09-14T03:44:20.774 INFO:tasks.rados.rados.0.smithi110.stdout:1957: finishing write tid 3 to smithi11075562-42 2024-09-14T03:44:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:1957: finishing write tid 4 to smithi11075562-42 2024-09-14T03:44:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1226 (ObjNum 624 snap 192 seq_num 624) dirty exists 2024-09-14T03:44:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:1957: left oid 42 (ObjNum 624 snap 192 seq_num 624) 2024-09-14T03:44:20.783 INFO:tasks.rados.rados.0.smithi110.stdout:1960: finishing write tid 1 to smithi11075562-49 2024-09-14T03:44:20.783 INFO:tasks.rados.rados.0.smithi110.stdout:1960: finishing write tid 2 to smithi11075562-49 2024-09-14T03:44:20.786 INFO:tasks.rados.rados.0.smithi110.stdout:1960: finishing write tid 3 to smithi11075562-49 2024-09-14T03:44:20.789 INFO:tasks.rados.rados.0.smithi110.stdout:1960: finishing write tid 4 to smithi11075562-49 2024-09-14T03:44:20.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1233 (ObjNum 625 snap 192 seq_num 625) dirty exists 2024-09-14T03:44:20.789 INFO:tasks.rados.rados.0.smithi110.stdout:1960: left oid 49 (ObjNum 625 snap 192 seq_num 625) 2024-09-14T03:44:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:20 smithi110 ceph-mon[88262]: pgmap v319: 121 pgs: 121 active+clean; 227 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 MiB/s wr, 10 op/s; 26 MiB/s, 6 objects/s recovering 2024-09-14T03:44:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:20 smithi110 ceph-mon[88262]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T03:44:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:20 smithi110 ceph-mon[86423]: pgmap v319: 121 pgs: 121 active+clean; 227 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 MiB/s wr, 10 op/s; 26 MiB/s, 6 objects/s recovering 2024-09-14T03:44:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:20 smithi110 ceph-mon[86423]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T03:44:21.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:20 smithi195 ceph-mon[82388]: pgmap v319: 121 pgs: 121 active+clean; 227 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 MiB/s wr, 10 op/s; 26 MiB/s, 6 objects/s recovering 2024-09-14T03:44:21.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:20 smithi195 ceph-mon[82388]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T03:44:21.773 INFO:tasks.rados.rados.0.smithi110.stdout:1957: done (4 left) 2024-09-14T03:44:21.773 INFO:tasks.rados.rados.0.smithi110.stdout:1958: done (3 left) 2024-09-14T03:44:21.773 INFO:tasks.rados.rados.0.smithi110.stdout:1960: done (2 left) 2024-09-14T03:44:21.773 INFO:tasks.rados.rados.0.smithi110.stdout:1961: done (1 left) 2024-09-14T03:44:21.773 INFO:tasks.rados.rados.0.smithi110.stdout:1962: done (0 left) 2024-09-14T03:44:21.773 INFO:tasks.rados.rados.0.smithi110.stdout:1963: setattr oid 37 current snap is 193 2024-09-14T03:44:21.774 INFO:tasks.rados.rados.0.smithi110.stdout:1964: copy_from oid 34 from oid 33 current snap is 193 2024-09-14T03:44:21.774 INFO:tasks.rados.rados.0.smithi110.stdout:1965: copy_from oid 16 from oid 43 current snap is 193 2024-09-14T03:44:21.774 INFO:tasks.rados.rados.0.smithi110.stdout:1966: rollback oid 13 current snap is 193 2024-09-14T03:44:21.774 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 192 2024-09-14T03:44:21.775 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 193 2024-09-14T03:44:21.775 INFO:tasks.rados.rados.0.smithi110.stdout:1967: seq_num 626 ranges {9715712=491520,10207232=401408} 2024-09-14T03:44:21.831 INFO:tasks.rados.rados.0.smithi110.stdout:1967: writing smithi11075562-24 from 9715712 to 10207232 tid 1 2024-09-14T03:44:21.837 INFO:tasks.rados.rados.0.smithi110.stdout:1967: writing smithi11075562-24 from 10207232 to 10608640 tid 2 2024-09-14T03:44:21.837 INFO:tasks.rados.rados.0.smithi110.stdout:1968: copy_from oid 38 from oid 11 current snap is 193 2024-09-14T03:44:21.837 INFO:tasks.rados.rados.0.smithi110.stdout:1966: finishing rollback tid 0 to smithi11075562-13 2024-09-14T03:44:21.837 INFO:tasks.rados.rados.0.smithi110.stdout:1966: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:44:21.838 INFO:tasks.rados.rados.0.smithi110.stdout:1966: finishing rollback tid 2 to smithi11075562-13 2024-09-14T03:44:21.838 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1239 (ObjNum 550 snap 167 seq_num 550) dirty exists 2024-09-14T03:44:21.838 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1428 (ObjNum 592 snap 181 seq_num 592) dirty exists 2024-09-14T03:44:21.838 INFO:tasks.rados.rados.0.smithi110.stdout:1965: finishing copy_from racing read to smithi11075562-16 2024-09-14T03:44:21.838 INFO:tasks.rados.rados.0.smithi110.stdout:1965: finishing copy_from to smithi11075562-16 2024-09-14T03:44:21.838 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 1083 (ObjNum 613 snap 188 seq_num 613) dirty exists 2024-09-14T03:44:21.838 INFO:tasks.rados.rados.0.smithi110.stdout:1964: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:44:21.838 INFO:tasks.rados.rados.0.smithi110.stdout:1964: finishing copy_from to smithi11075562-34 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1043 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1963: done (5 left) 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1964: done (4 left) 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1965: done (3 left) 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1966: done (2 left) 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1969: rollback oid 26 current snap is 193 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 190 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1970: snap_create 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1969: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1969: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1967: finishing write tid 1 to smithi11075562-24 2024-09-14T03:44:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:1969: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:44:21.840 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 776 (ObjNum 622 snap 190 seq_num 622) dirty exists 2024-09-14T03:44:21.840 INFO:tasks.rados.rados.0.smithi110.stdout:1967: finishing write tid 2 to smithi11075562-24 2024-09-14T03:44:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:1967: finishing write tid 3 to smithi11075562-24 2024-09-14T03:44:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:1967: finishing write tid 4 to smithi11075562-24 2024-09-14T03:44:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 923 (ObjNum 626 snap 193 seq_num 626) dirty exists 2024-09-14T03:44:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:1967: left oid 24 (ObjNum 626 snap 193 seq_num 626) 2024-09-14T03:44:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:1968: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:44:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:1968: finishing copy_from to smithi11075562-38 2024-09-14T03:44:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1080 (ObjNum 604 snap 186 seq_num 604) dirty exists 2024-09-14T03:44:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:21 smithi110 ceph-mon[88262]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T03:44:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:21 smithi110 ceph-mon[86423]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T03:44:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:44:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:44:21] "GET /metrics HTTP/1.1" 200 36108 "" "Prometheus/2.51.0" 2024-09-14T03:44:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:21 smithi195 ceph-mon[82388]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T03:44:22.779 INFO:tasks.rados.rados.0.smithi110.stdout:1967: done (3 left) 2024-09-14T03:44:22.779 INFO:tasks.rados.rados.0.smithi110.stdout:1968: done (2 left) 2024-09-14T03:44:22.779 INFO:tasks.rados.rados.0.smithi110.stdout:1969: done (1 left) 2024-09-14T03:44:22.779 INFO:tasks.rados.rados.0.smithi110.stdout:1970: done (0 left) 2024-09-14T03:44:22.779 INFO:tasks.rados.rados.0.smithi110.stdout:1971: rmattr oid 9 current snap is 194 2024-09-14T03:44:22.779 INFO:tasks.rados.rados.0.smithi110.stdout:1971: done (0 left) 2024-09-14T03:44:22.779 INFO:tasks.rados.rados.0.smithi110.stdout:1972: copy_from oid 8 from oid 42 current snap is 194 2024-09-14T03:44:22.779 INFO:tasks.rados.rados.0.smithi110.stdout:1973: delete oid 3 current snap is 194 2024-09-14T03:44:22.785 INFO:tasks.rados.rados.0.smithi110.stdout:1973: done (1 left) 2024-09-14T03:44:22.785 INFO:tasks.rados.rados.0.smithi110.stdout:1974: rollback oid 25 current snap is 194 2024-09-14T03:44:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 25 to 193 2024-09-14T03:44:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:1975: snap_create 2024-09-14T03:44:22.789 INFO:tasks.rados.rados.0.smithi110.stdout:1972: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:44:22.790 INFO:tasks.rados.rados.0.smithi110.stdout:1974: finishing rollback tid 0 to smithi11075562-25 2024-09-14T03:44:22.791 INFO:tasks.rados.rados.0.smithi110.stdout:1974: finishing rollback tid 1 to smithi11075562-25 2024-09-14T03:44:22.792 INFO:tasks.rados.rados.0.smithi110.stdout:1972: finishing copy_from to smithi11075562-8 2024-09-14T03:44:22.792 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1226 (ObjNum 624 snap 192 seq_num 624) dirty exists 2024-09-14T03:44:22.792 INFO:tasks.rados.rados.0.smithi110.stdout:1974: finishing rollback tid 2 to smithi11075562-25 2024-09-14T03:44:22.792 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1432 (ObjNum 602 snap 184 seq_num 602) dirty exists 2024-09-14T03:44:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:22 smithi110 ceph-mon[88262]: pgmap v321: 121 pgs: 121 active+clean; 222 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 764 B/s rd, 627 KiB/s wr, 7 op/s; 2.5 MiB/s, 0 objects/s recovering 2024-09-14T03:44:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:22 smithi110 ceph-mon[88262]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T03:44:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:22 smithi110 ceph-mon[86423]: pgmap v321: 121 pgs: 121 active+clean; 222 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 764 B/s rd, 627 KiB/s wr, 7 op/s; 2.5 MiB/s, 0 objects/s recovering 2024-09-14T03:44:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:22 smithi110 ceph-mon[86423]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T03:44:23.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:22 smithi195 ceph-mon[82388]: pgmap v321: 121 pgs: 121 active+clean; 222 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 764 B/s rd, 627 KiB/s wr, 7 op/s; 2.5 MiB/s, 0 objects/s recovering 2024-09-14T03:44:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:22 smithi195 ceph-mon[82388]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T03:44:23.714 INFO:tasks.rados.rados.0.smithi110.stdout:1972: done (2 left) 2024-09-14T03:44:23.714 INFO:tasks.rados.rados.0.smithi110.stdout:1974: done (1 left) 2024-09-14T03:44:23.714 INFO:tasks.rados.rados.0.smithi110.stdout:1975: done (0 left) 2024-09-14T03:44:23.715 INFO:tasks.rados.rados.0.smithi110.stdout:1976: read oid 30 snap 192 2024-09-14T03:44:23.715 INFO:tasks.rados.rados.0.smithi110.stdout:1976: expect deleted 2024-09-14T03:44:23.715 INFO:tasks.rados.rados.0.smithi110.stdout:1977: rollback oid 17 current snap is 195 2024-09-14T03:44:23.715 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 192 2024-09-14T03:44:23.715 INFO:tasks.rados.rados.0.smithi110.stdout:1978: read oid 14 snap 194 2024-09-14T03:44:23.715 INFO:tasks.rados.rados.0.smithi110.stdout:1978: expect (ObjNum 603 snap 185 seq_num 603) 2024-09-14T03:44:23.715 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 195 2024-09-14T03:44:23.715 INFO:tasks.rados.rados.0.smithi110.stdout:1979: seq_num 628 ranges {0=524288,524288=401408} 2024-09-14T03:44:23.717 INFO:tasks.rados.rados.0.smithi110.stdout:1979: writing smithi11075562-45 from 0 to 524288 tid 1 2024-09-14T03:44:23.722 INFO:tasks.rados.rados.0.smithi110.stdout:1979: writing smithi11075562-45 from 524288 to 925696 tid 2 2024-09-14T03:44:23.722 INFO:tasks.rados.rados.0.smithi110.stdout:1980: rmattr oid 7 current snap is 195 2024-09-14T03:44:23.722 INFO:tasks.rados.rados.0.smithi110.stdout:1980: done (4 left) 2024-09-14T03:44:23.722 INFO:tasks.rados.rados.0.smithi110.stdout:1981: copy_from oid 33 from oid 36 current snap is 195 2024-09-14T03:44:23.722 INFO:tasks.rados.rados.0.smithi110.stdout:1977: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:44:23.722 INFO:tasks.rados.rados.0.smithi110.stdout:1977: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:44:23.723 INFO:tasks.rados.rados.0.smithi110.stdout:1977: finishing rollback tid 2 to smithi11075562-17 2024-09-14T03:44:23.723 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1084 (ObjNum 551 snap 167 seq_num 551) dirty exists 2024-09-14T03:44:23.723 INFO:tasks.rados.rados.0.smithi110.stdout:1976: done (4 left) 2024-09-14T03:44:23.723 INFO:tasks.rados.rados.0.smithi110.stdout:1977: done (3 left) 2024-09-14T03:44:23.723 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 195 2024-09-14T03:44:23.723 INFO:tasks.rados.rados.0.smithi110.stdout:1982: seq_num 629 ranges {1785856=540672,2326528=401408} 2024-09-14T03:44:23.734 INFO:tasks.rados.rados.0.smithi110.stdout:1982: writing smithi11075562-20 from 1785856 to 2326528 tid 1 2024-09-14T03:44:23.737 INFO:tasks.rados.rados.0.smithi110.stdout:1982: writing smithi11075562-20 from 2326528 to 2727936 tid 2 2024-09-14T03:44:23.737 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 195 2024-09-14T03:44:23.737 INFO:tasks.rados.rados.0.smithi110.stdout:1979: finishing write tid 1 to smithi11075562-45 2024-09-14T03:44:23.737 INFO:tasks.rados.rados.0.smithi110.stdout:1983: seq_num 630 ranges {2850816=557056,3407872=401408} 2024-09-14T03:44:23.756 INFO:tasks.rados.rados.0.smithi110.stdout:1983: writing smithi11075562-15 from 2850816 to 3407872 tid 1 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:1983: writing smithi11075562-15 from 3407872 to 3809280 tid 2 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 195 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:1979: finishing write tid 2 to smithi11075562-45 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:1979: finishing write tid 3 to smithi11075562-45 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:1979: finishing write tid 4 to smithi11075562-45 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 1435 (ObjNum 628 snap 195 seq_num 628) dirty exists 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:1979: left oid 45 (ObjNum 628 snap 195 seq_num 628) 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:1981: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:1981: finishing copy_from to smithi11075562-33 2024-09-14T03:44:23.759 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1414 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-09-14T03:44:23.760 INFO:tasks.rados.rados.0.smithi110.stdout:1982: finishing write tid 1 to smithi11075562-20 2024-09-14T03:44:23.760 INFO:tasks.rados.rados.0.smithi110.stdout:1982: finishing write tid 2 to smithi11075562-20 2024-09-14T03:44:23.760 INFO:tasks.rados.rados.0.smithi110.stdout:1982: finishing write tid 3 to smithi11075562-20 2024-09-14T03:44:23.760 INFO:tasks.rados.rados.0.smithi110.stdout:1982: finishing write tid 4 to smithi11075562-20 2024-09-14T03:44:23.760 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1417 (ObjNum 629 snap 195 seq_num 629) dirty exists 2024-09-14T03:44:23.760 INFO:tasks.rados.rados.0.smithi110.stdout:1982: left oid 20 (ObjNum 629 snap 195 seq_num 629) 2024-09-14T03:44:23.760 INFO:tasks.rados.rados.0.smithi110.stdout:1984: seq_num 631 ranges {6946816=573440,7520256=401408} 2024-09-14T03:44:23.800 INFO:tasks.rados.rados.0.smithi110.stdout:1984: writing smithi11075562-27 from 6946816 to 7520256 tid 1 2024-09-14T03:44:23.803 INFO:tasks.rados.rados.0.smithi110.stdout:1984: writing smithi11075562-27 from 7520256 to 7921664 tid 2 2024-09-14T03:44:23.803 INFO:tasks.rados.rados.0.smithi110.stdout:1979: done (5 left) 2024-09-14T03:44:23.803 INFO:tasks.rados.rados.0.smithi110.stdout:1981: done (4 left) 2024-09-14T03:44:23.803 INFO:tasks.rados.rados.0.smithi110.stdout:1982: done (3 left) 2024-09-14T03:44:23.803 INFO:tasks.rados.rados.0.smithi110.stdout:1985: snap_create 2024-09-14T03:44:23.803 INFO:tasks.rados.rados.0.smithi110.stdout:1983: finishing write tid 1 to smithi11075562-15 2024-09-14T03:44:23.803 INFO:tasks.rados.rados.0.smithi110.stdout:1978: expect (ObjNum 603 snap 185 seq_num 603) 2024-09-14T03:44:24.148 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:23 smithi195 ceph-mon[82388]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T03:44:24.148 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:23 smithi195 ceph-mon[82388]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T03:44:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:23 smithi110 ceph-mon[88262]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T03:44:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:23 smithi110 ceph-mon[88262]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T03:44:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:23 smithi110 ceph-mon[86423]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T03:44:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:23 smithi110 ceph-mon[86423]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T03:44:24.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:24 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:24.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:44:24.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:24 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.195\", device_class=\"ssd\", hostname=\"smithi195\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.195\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:1983: finishing write tid 2 to smithi11075562-15 2024-09-14T03:44:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:1983: finishing write tid 3 to smithi11075562-15 2024-09-14T03:44:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:1983: finishing write tid 4 to smithi11075562-15 2024-09-14T03:44:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1229 (ObjNum 630 snap 195 seq_num 630) dirty exists 2024-09-14T03:44:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:1983: left oid 15 (ObjNum 630 snap 195 seq_num 630) 2024-09-14T03:44:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:1978: done (3 left) 2024-09-14T03:44:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:1983: done (2 left) 2024-09-14T03:44:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:1985: done (1 left) 2024-09-14T03:44:24.811 INFO:tasks.rados.rados.0.smithi110.stdout:1986: delete oid 40 current snap is 196 2024-09-14T03:44:24.811 INFO:tasks.rados.rados.0.smithi110.stdout:1984: finishing write tid 1 to smithi11075562-27 2024-09-14T03:44:24.811 INFO:tasks.rados.rados.0.smithi110.stdout:1984: finishing write tid 2 to smithi11075562-27 2024-09-14T03:44:24.811 INFO:tasks.rados.rados.0.smithi110.stdout:1984: finishing write tid 3 to smithi11075562-27 2024-09-14T03:44:24.811 INFO:tasks.rados.rados.0.smithi110.stdout:1984: finishing write tid 4 to smithi11075562-27 2024-09-14T03:44:24.811 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1197 (ObjNum 631 snap 195 seq_num 631) dirty exists 2024-09-14T03:44:24.811 INFO:tasks.rados.rados.0.smithi110.stdout:1984: left oid 27 (ObjNum 631 snap 195 seq_num 631) 2024-09-14T03:44:24.812 INFO:tasks.rados.rados.0.smithi110.stdout:1984: done (1 left) 2024-09-14T03:44:24.812 INFO:tasks.rados.rados.0.smithi110.stdout:1986: done (0 left) 2024-09-14T03:44:24.812 INFO:tasks.rados.rados.0.smithi110.stdout:1987: read oid 17 snap 195 2024-09-14T03:44:24.813 INFO:tasks.rados.rados.0.smithi110.stdout:1987: expect (ObjNum 551 snap 167 seq_num 551) 2024-09-14T03:44:24.813 INFO:tasks.rados.rados.0.smithi110.stdout:1988: read oid 45 snap 194 2024-09-14T03:44:24.813 INFO:tasks.rados.rados.0.smithi110.stdout:1988: expect deleted 2024-09-14T03:44:24.813 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 196 2024-09-14T03:44:24.813 INFO:tasks.rados.rados.0.smithi110.stdout:1989: seq_num 633 ranges {4333568=606208,4939776=401408} 2024-09-14T03:44:24.839 INFO:tasks.rados.rados.0.smithi110.stdout:1989: writing smithi11075562-29 from 4333568 to 4939776 tid 1 2024-09-14T03:44:24.842 INFO:tasks.rados.rados.0.smithi110.stdout:1989: writing smithi11075562-29 from 4939776 to 5341184 tid 2 2024-09-14T03:44:24.842 INFO:tasks.rados.rados.0.smithi110.stdout:1990: snap_remove snap 192 2024-09-14T03:44:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:24 smithi110 ceph-mon[88262]: pgmap v325: 121 pgs: 121 active+clean; 190 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 552 KiB/s wr, 8 op/s 2024-09-14T03:44:25.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:24 smithi110 ceph-mon[86423]: pgmap v325: 121 pgs: 121 active+clean; 190 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 552 KiB/s wr, 8 op/s 2024-09-14T03:44:25.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:24 smithi195 ceph-mon[82388]: pgmap v325: 121 pgs: 121 active+clean; 190 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 552 KiB/s wr, 8 op/s 2024-09-14T03:44:25.789 INFO:tasks.rados.rados.0.smithi110.stdout:1990: done (3 left) 2024-09-14T03:44:25.789 INFO:tasks.rados.rados.0.smithi110.stdout:1991: delete oid 3 current snap is 196 2024-09-14T03:44:25.789 INFO:tasks.rados.rados.0.smithi110.stdout:1987: expect (ObjNum 551 snap 167 seq_num 551) 2024-09-14T03:44:25.846 INFO:tasks.rados.rados.0.smithi110.stdout:1989: finishing write tid 1 to smithi11075562-29 2024-09-14T03:44:25.846 INFO:tasks.rados.rados.0.smithi110.stdout:1989: finishing write tid 2 to smithi11075562-29 2024-09-14T03:44:25.846 INFO:tasks.rados.rados.0.smithi110.stdout:1989: finishing write tid 3 to smithi11075562-29 2024-09-14T03:44:25.846 INFO:tasks.rados.rados.0.smithi110.stdout:1989: finishing write tid 4 to smithi11075562-29 2024-09-14T03:44:25.846 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1190 (ObjNum 633 snap 196 seq_num 633) dirty exists 2024-09-14T03:44:25.847 INFO:tasks.rados.rados.0.smithi110.stdout:1989: left oid 29 (ObjNum 633 snap 196 seq_num 633) 2024-09-14T03:44:25.847 INFO:tasks.rados.rados.0.smithi110.stdout:1987: done (3 left) 2024-09-14T03:44:25.847 INFO:tasks.rados.rados.0.smithi110.stdout:1988: done (2 left) 2024-09-14T03:44:25.847 INFO:tasks.rados.rados.0.smithi110.stdout:1989: done (1 left) 2024-09-14T03:44:25.847 INFO:tasks.rados.rados.0.smithi110.stdout:1991: done (0 left) 2024-09-14T03:44:25.847 INFO:tasks.rados.rados.0.smithi110.stdout:1992: setattr oid 44 current snap is 196 2024-09-14T03:44:25.847 INFO:tasks.rados.rados.0.smithi110.stdout:1993: rollback oid 18 current snap is 196 2024-09-14T03:44:25.848 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 18 to 194 2024-09-14T03:44:25.848 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 196 2024-09-14T03:44:25.848 INFO:tasks.rados.rados.0.smithi110.stdout:1994: seq_num 635 ranges {0=638976,638976=401408} 2024-09-14T03:44:25.851 INFO:tasks.rados.rados.0.smithi110.stdout:1994: writing smithi11075562-48 from 0 to 638976 tid 1 2024-09-14T03:44:25.855 INFO:tasks.rados.rados.0.smithi110.stdout:1994: writing smithi11075562-48 from 638976 to 1040384 tid 2 2024-09-14T03:44:25.855 INFO:tasks.rados.rados.0.smithi110.stdout:1995: read oid 1 snap -1 2024-09-14T03:44:25.855 INFO:tasks.rados.rados.0.smithi110.stdout:1995: expect (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:44:25.855 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 196 2024-09-14T03:44:25.855 INFO:tasks.rados.rados.0.smithi110.stdout:1993: finishing rollback tid 0 to smithi11075562-18 2024-09-14T03:44:25.856 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 400 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:44:25.856 INFO:tasks.rados.rados.0.smithi110.stdout:1993: finishing rollback tid 1 to smithi11075562-18 2024-09-14T03:44:25.856 INFO:tasks.rados.rados.0.smithi110.stdout:1993: finishing rollback tid 2 to smithi11075562-18 2024-09-14T03:44:25.856 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1094 (ObjNum 595 snap 181 seq_num 595) dirty exists 2024-09-14T03:44:25.856 INFO:tasks.rados.rados.0.smithi110.stdout:1996: seq_num 636 ranges {3809280=655360,4464640=401408} 2024-09-14T03:44:25.879 INFO:tasks.rados.rados.0.smithi110.stdout:1996: writing smithi11075562-15 from 3809280 to 4464640 tid 1 2024-09-14T03:44:25.882 INFO:tasks.rados.rados.0.smithi110.stdout:1996: writing smithi11075562-15 from 4464640 to 4866048 tid 2 2024-09-14T03:44:25.882 INFO:tasks.rados.rados.0.smithi110.stdout:1992: done (4 left) 2024-09-14T03:44:25.882 INFO:tasks.rados.rados.0.smithi110.stdout:1993: done (3 left) 2024-09-14T03:44:25.883 INFO:tasks.rados.rados.0.smithi110.stdout:1997: delete oid 35 current snap is 196 2024-09-14T03:44:25.883 INFO:tasks.rados.rados.0.smithi110.stdout:1994: finishing write tid 1 to smithi11075562-48 2024-09-14T03:44:25.883 INFO:tasks.rados.rados.0.smithi110.stdout:1994: finishing write tid 2 to smithi11075562-48 2024-09-14T03:44:25.883 INFO:tasks.rados.rados.0.smithi110.stdout:1994: finishing write tid 3 to smithi11075562-48 2024-09-14T03:44:25.883 INFO:tasks.rados.rados.0.smithi110.stdout:1994: finishing write tid 4 to smithi11075562-48 2024-09-14T03:44:25.883 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1087 (ObjNum 635 snap 196 seq_num 635) dirty exists 2024-09-14T03:44:25.883 INFO:tasks.rados.rados.0.smithi110.stdout:1994: left oid 48 (ObjNum 635 snap 196 seq_num 635) 2024-09-14T03:44:25.883 INFO:tasks.rados.rados.0.smithi110.stdout:1995: expect (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:44:25.944 INFO:tasks.rados.rados.0.smithi110.stdout:1996: finishing write tid 1 to smithi11075562-15 2024-09-14T03:44:25.944 INFO:tasks.rados.rados.0.smithi110.stdout:1996: finishing write tid 2 to smithi11075562-15 2024-09-14T03:44:25.944 INFO:tasks.rados.rados.0.smithi110.stdout:1996: finishing write tid 3 to smithi11075562-15 2024-09-14T03:44:25.944 INFO:tasks.rados.rados.0.smithi110.stdout:1996: finishing write tid 4 to smithi11075562-15 2024-09-14T03:44:25.944 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1232 (ObjNum 636 snap 196 seq_num 636) dirty exists 2024-09-14T03:44:25.945 INFO:tasks.rados.rados.0.smithi110.stdout:1996: left oid 15 (ObjNum 636 snap 196 seq_num 636) 2024-09-14T03:44:25.945 INFO:tasks.rados.rados.0.smithi110.stdout:1994: done (3 left) 2024-09-14T03:44:25.945 INFO:tasks.rados.rados.0.smithi110.stdout:1995: done (2 left) 2024-09-14T03:44:25.945 INFO:tasks.rados.rados.0.smithi110.stdout:1996: done (1 left) 2024-09-14T03:44:25.945 INFO:tasks.rados.rados.0.smithi110.stdout:1997: done (0 left) 2024-09-14T03:44:25.945 INFO:tasks.rados.rados.0.smithi110.stdout:1998: snap_create 2024-09-14T03:44:26.003 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:25 smithi195 ceph-mon[82388]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T03:44:26.003 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:25 smithi195 ceph-mon[82388]: pgmap v327: 121 pgs: 121 active+clean; 189 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 3.4 MiB/s wr, 10 op/s 2024-09-14T03:44:26.004 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:26.004 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:25 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:25 smithi110 ceph-mon[88262]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T03:44:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:25 smithi110 ceph-mon[88262]: pgmap v327: 121 pgs: 121 active+clean; 189 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 3.4 MiB/s wr, 10 op/s 2024-09-14T03:44:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:25 smithi110 ceph-mon[86423]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T03:44:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:25 smithi110 ceph-mon[86423]: pgmap v327: 121 pgs: 121 active+clean; 189 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 3.4 MiB/s wr, 10 op/s 2024-09-14T03:44:26.794 INFO:tasks.rados.rados.0.smithi110.stdout:1998: done (0 left) 2024-09-14T03:44:26.794 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 197 2024-09-14T03:44:26.794 INFO:tasks.rados.rados.0.smithi110.stdout:1999: seq_num 638 ranges {0=688128,688128=401408} 2024-09-14T03:44:26.798 INFO:tasks.rados.rados.0.smithi110.stdout:1999: writing smithi11075562-3 from 0 to 688128 tid 1 2024-09-14T03:44:26.801 INFO:tasks.rados.rados.0.smithi110.stdout:1999: writing smithi11075562-3 from 688128 to 1089536 tid 2 2024-09-14T03:44:26.801 INFO:tasks.rados.rados.0.smithi110.stdout:2000: rollback oid 11 current snap is 197 2024-09-14T03:44:26.801 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 193 2024-09-14T03:44:26.801 INFO:tasks.rados.rados.0.smithi110.stdout:2001: rollback oid 26 current snap is 197 2024-09-14T03:44:26.801 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 195 2024-09-14T03:44:26.801 INFO:tasks.rados.rados.0.smithi110.stdout:2002: rollback oid 19 current snap is 197 2024-09-14T03:44:26.802 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 194 2024-09-14T03:44:26.802 INFO:tasks.rados.rados.0.smithi110.stdout:2003: snap_create 2024-09-14T03:44:26.805 INFO:tasks.rados.rados.0.smithi110.stdout:2002: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:44:26.805 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 0 (ObjNum 544235886 snap 171128881 seq_num 304) dirty dne 2024-09-14T03:44:26.805 INFO:tasks.rados.rados.0.smithi110.stdout:2000: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:44:26.805 INFO:tasks.rados.rados.0.smithi110.stdout:2001: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:44:26.806 INFO:tasks.rados.rados.0.smithi110.stdout:2001: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:44:26.807 INFO:tasks.rados.rados.0.smithi110.stdout:2001: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:44:26.807 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 779 (ObjNum 622 snap 190 seq_num 622) dirty exists 2024-09-14T03:44:26.807 INFO:tasks.rados.rados.0.smithi110.stdout:2000: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:44:26.808 INFO:tasks.rados.rados.0.smithi110.stdout:2000: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:44:26.808 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1090 (ObjNum 604 snap 186 seq_num 604) dirty exists 2024-09-14T03:44:26.808 INFO:tasks.rados.rados.0.smithi110.stdout:1999: finishing write tid 1 to smithi11075562-3 2024-09-14T03:44:26.808 INFO:tasks.rados.rados.0.smithi110.stdout:1999: finishing write tid 2 to smithi11075562-3 2024-09-14T03:44:26.808 INFO:tasks.rados.rados.0.smithi110.stdout:1999: finishing write tid 3 to smithi11075562-3 2024-09-14T03:44:26.809 INFO:tasks.rados.rados.0.smithi110.stdout:1999: finishing write tid 4 to smithi11075562-3 2024-09-14T03:44:26.809 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 727 (ObjNum 638 snap 197 seq_num 638) dirty exists 2024-09-14T03:44:26.809 INFO:tasks.rados.rados.0.smithi110.stdout:1999: left oid 3 (ObjNum 638 snap 197 seq_num 638) 2024-09-14T03:44:27.075 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:26 smithi110 ceph-mon[86423]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T03:44:27.075 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:26 smithi110 ceph-mon[88262]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T03:44:27.075 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:44:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:26 smithi195 ceph-mon[82388]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T03:44:27.209 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:26 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi110\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi110\", job=\"node\", machine=\"x86_64\", nodename=\"smithi110\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:27.623 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:44:27.623 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (5m) 51s ago 9m 19.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:44:27.623 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (4m) 18s ago 9m 84.6M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:44:27.623 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (10m) 51s ago 10m 20.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:44:27.623 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (10m) 18s ago 10m 17.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:44:27.623 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (4m) 18s ago 13m 454M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:44:27.623 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (4m) 51s ago 15m 525M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:44:27.623 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (3m) 51s ago 15m 55.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (3m) 18s ago 14m 43.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (3m) 51s ago 14m 49.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (5m) 51s ago 8m 9.92M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (4m) 18s ago 8m 11.0M - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (2m) 51s ago 13m 218M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (2m) 51s ago 13m 117M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (89s) 51s ago 12m 101M 2480M 19.3.0-4907-ga2782315 8801d2529e43 0cdc58c38711 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (55s) 51s ago 12m 14.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2283ebcb5f10 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (22s) 18s ago 12m 14.5M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6921b0a30047 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (11m) 18s ago 11m 298M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 d7b0180ede9d 2024-09-14T03:44:27.624 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (11m) 18s ago 11m 214M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:44:27.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (10m) 18s ago 10m 174M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:44:27.625 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (4m) 18s ago 9m 52.7M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:44:27.799 INFO:tasks.rados.rados.0.smithi110.stdout:1999: done (4 left) 2024-09-14T03:44:27.799 INFO:tasks.rados.rados.0.smithi110.stdout:2000: done (3 left) 2024-09-14T03:44:27.799 INFO:tasks.rados.rados.0.smithi110.stdout:2001: done (2 left) 2024-09-14T03:44:27.799 INFO:tasks.rados.rados.0.smithi110.stdout:2002: done (1 left) 2024-09-14T03:44:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:2003: done (0 left) 2024-09-14T03:44:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 198 2024-09-14T03:44:27.800 INFO:tasks.rados.rados.0.smithi110.stdout:2004: seq_num 639 ranges {524288=704512,1228800=401408} 2024-09-14T03:44:27.806 INFO:tasks.rados.rados.0.smithi110.stdout:2004: writing smithi11075562-38 from 524288 to 1228800 tid 1 2024-09-14T03:44:27.808 INFO:tasks.rados.rados.0.smithi110.stdout:2004: writing smithi11075562-38 from 1228800 to 1630208 tid 2 2024-09-14T03:44:27.808 INFO:tasks.rados.rados.0.smithi110.stdout:2005: snap_create 2024-09-14T03:44:27.812 INFO:tasks.rados.rados.0.smithi110.stdout:2004: finishing write tid 1 to smithi11075562-38 2024-09-14T03:44:27.813 INFO:tasks.rados.rados.0.smithi110.stdout:2004: finishing write tid 2 to smithi11075562-38 2024-09-14T03:44:27.813 INFO:tasks.rados.rados.0.smithi110.stdout:2004: finishing write tid 3 to smithi11075562-38 2024-09-14T03:44:27.813 INFO:tasks.rados.rados.0.smithi110.stdout:2004: finishing write tid 4 to smithi11075562-38 2024-09-14T03:44:27.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1193 (ObjNum 639 snap 198 seq_num 639) dirty exists 2024-09-14T03:44:27.813 INFO:tasks.rados.rados.0.smithi110.stdout:2004: left oid 38 (ObjNum 639 snap 198 seq_num 639) 2024-09-14T03:44:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[86423]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T03:44:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[86423]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[86423]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[86423]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[86423]: pgmap v330: 121 pgs: 121 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.6 MiB/s wr, 20 op/s 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[88262]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[88262]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[88262]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[88262]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:27.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:27 smithi110 ceph-mon[88262]: pgmap v330: 121 pgs: 121 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.6 MiB/s wr, 20 op/s 2024-09-14T03:44:27.970 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:44:27.970 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:44:27.970 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:44:27.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:44:27.970 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:44:27.970 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:44:27.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:44:27.970 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3, 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 5, 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 10 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:44:27.971 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:44:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:27 smithi195 ceph-mon[82388]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T03:44:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:27 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:27 smithi195 ceph-mon[82388]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:27 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:27 smithi195 ceph-mon[82388]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:27 smithi195 ceph-mon[82388]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:27 smithi195 ceph-mon[82388]: pgmap v330: 121 pgs: 121 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.6 MiB/s wr, 20 op/s 2024-09-14T03:44:28.253 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:44:28.253 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "10/20 daemons upgraded", 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:44:28.254 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:44:28.715 INFO:tasks.rados.rados.0.smithi110.stdout:2004: done (1 left) 2024-09-14T03:44:28.716 INFO:tasks.rados.rados.0.smithi110.stdout:2005: done (0 left) 2024-09-14T03:44:28.716 INFO:tasks.rados.rados.0.smithi110.stdout:2006: setattr oid 29 current snap is 199 2024-09-14T03:44:28.717 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 199 2024-09-14T03:44:28.717 INFO:tasks.rados.rados.0.smithi110.stdout:2007: seq_num 640 ranges {7241728=729088,7970816=401408} 2024-09-14T03:44:28.759 INFO:tasks.rados.rados.0.smithi110.stdout:2007: writing smithi11075562-31 from 7241728 to 7970816 tid 1 2024-09-14T03:44:28.762 INFO:tasks.rados.rados.0.smithi110.stdout:2007: writing smithi11075562-31 from 7970816 to 8372224 tid 2 2024-09-14T03:44:28.762 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 199 2024-09-14T03:44:28.762 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1194 (ObjNum 633 snap 196 seq_num 633) dirty exists 2024-09-14T03:44:28.762 INFO:tasks.rados.rados.0.smithi110.stdout:2008: seq_num 641 ranges {983040=745472,1728512=401408} 2024-09-14T03:44:28.771 INFO:tasks.rados.rados.0.smithi110.stdout:2008: writing smithi11075562-46 from 983040 to 1728512 tid 1 2024-09-14T03:44:28.774 INFO:tasks.rados.rados.0.smithi110.stdout:2008: writing smithi11075562-46 from 1728512 to 2129920 tid 2 2024-09-14T03:44:28.774 INFO:tasks.rados.rados.0.smithi110.stdout:2006: done (2 left) 2024-09-14T03:44:28.774 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 199 2024-09-14T03:44:28.774 INFO:tasks.rados.rados.0.smithi110.stdout:2007: finishing write tid 1 to smithi11075562-31 2024-09-14T03:44:28.774 INFO:tasks.rados.rados.0.smithi110.stdout:2007: finishing write tid 2 to smithi11075562-31 2024-09-14T03:44:28.774 INFO:tasks.rados.rados.0.smithi110.stdout:2007: finishing write tid 3 to smithi11075562-31 2024-09-14T03:44:28.775 INFO:tasks.rados.rados.0.smithi110.stdout:2007: finishing write tid 4 to smithi11075562-31 2024-09-14T03:44:28.775 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1155 (ObjNum 640 snap 199 seq_num 640) dirty exists 2024-09-14T03:44:28.775 INFO:tasks.rados.rados.0.smithi110.stdout:2007: left oid 31 (ObjNum 640 snap 199 seq_num 640) 2024-09-14T03:44:28.775 INFO:tasks.rados.rados.0.smithi110.stdout:2009: seq_num 642 ranges {7143424=761856,7905280=401408} 2024-09-14T03:44:28.816 INFO:tasks.rados.rados.0.smithi110.stdout:2009: writing smithi11075562-25 from 7143424 to 7905280 tid 1 2024-09-14T03:44:28.819 INFO:tasks.rados.rados.0.smithi110.stdout:2009: writing smithi11075562-25 from 7905280 to 8306688 tid 2 2024-09-14T03:44:28.819 INFO:tasks.rados.rados.0.smithi110.stdout:2007: done (2 left) 2024-09-14T03:44:28.819 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 199 2024-09-14T03:44:28.819 INFO:tasks.rados.rados.0.smithi110.stdout:2010: seq_num 643 ranges {860160=778240,1638400=401408} 2024-09-14T03:44:28.827 INFO:tasks.rados.rados.0.smithi110.stdout:2010: writing smithi11075562-42 from 860160 to 1638400 tid 1 2024-09-14T03:44:28.830 INFO:tasks.rados.rados.0.smithi110.stdout:2010: writing smithi11075562-42 from 1638400 to 2039808 tid 2 2024-09-14T03:44:28.830 INFO:tasks.rados.rados.0.smithi110.stdout:2011: rmattr oid 18 current snap is 199 2024-09-14T03:44:28.830 INFO:tasks.rados.rados.0.smithi110.stdout:2008: finishing write tid 1 to smithi11075562-46 2024-09-14T03:44:28.830 INFO:tasks.rados.rados.0.smithi110.stdout:2008: finishing write tid 2 to smithi11075562-46 2024-09-14T03:44:28.830 INFO:tasks.rados.rados.0.smithi110.stdout:2012: setattr oid 47 current snap is 199 2024-09-14T03:44:28.830 INFO:tasks.rados.rados.0.smithi110.stdout:2008: finishing write tid 3 to smithi11075562-46 2024-09-14T03:44:28.830 INFO:tasks.rados.rados.0.smithi110.stdout:2008: finishing write tid 4 to smithi11075562-46 2024-09-14T03:44:28.831 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1420 (ObjNum 641 snap 199 seq_num 641) dirty exists 2024-09-14T03:44:28.831 INFO:tasks.rados.rados.0.smithi110.stdout:2008: left oid 46 (ObjNum 641 snap 199 seq_num 641) 2024-09-14T03:44:28.831 INFO:tasks.rados.rados.0.smithi110.stdout:2009: finishing write tid 1 to smithi11075562-25 2024-09-14T03:44:28.831 INFO:tasks.rados.rados.0.smithi110.stdout:2009: finishing write tid 2 to smithi11075562-25 2024-09-14T03:44:28.831 INFO:tasks.rados.rados.0.smithi110.stdout:2009: finishing write tid 3 to smithi11075562-25 2024-09-14T03:44:28.831 INFO:tasks.rados.rados.0.smithi110.stdout:2009: finishing write tid 4 to smithi11075562-25 2024-09-14T03:44:28.831 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1443 (ObjNum 642 snap 199 seq_num 642) dirty exists 2024-09-14T03:44:28.831 INFO:tasks.rados.rados.0.smithi110.stdout:2009: left oid 25 (ObjNum 642 snap 199 seq_num 642) 2024-09-14T03:44:28.832 INFO:tasks.rados.rados.0.smithi110.stdout:2008: done (4 left) 2024-09-14T03:44:28.832 INFO:tasks.rados.rados.0.smithi110.stdout:2009: done (3 left) 2024-09-14T03:44:28.832 INFO:tasks.rados.rados.0.smithi110.stdout:2013: rollback oid 39 current snap is 199 2024-09-14T03:44:28.832 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 198 2024-09-14T03:44:28.832 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 199 2024-09-14T03:44:28.832 INFO:tasks.rados.rados.0.smithi110.stdout:2014: seq_num 644 ranges {860160=794624,1654784=401408} 2024-09-14T03:44:28.840 INFO:tasks.rados.rados.0.smithi110.stdout:2014: writing smithi11075562-8 from 860160 to 1654784 tid 1 2024-09-14T03:44:28.843 INFO:tasks.rados.rados.0.smithi110.stdout:2014: writing smithi11075562-8 from 1654784 to 2056192 tid 2 2024-09-14T03:44:28.843 INFO:tasks.rados.rados.0.smithi110.stdout:2015: rollback oid 28 current snap is 199 2024-09-14T03:44:28.843 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 28 to 197 2024-09-14T03:44:28.843 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1097 (ObjNum 595 snap 181 seq_num 595) dirty exists 2024-09-14T03:44:28.843 INFO:tasks.rados.rados.0.smithi110.stdout:2010: finishing write tid 1 to smithi11075562-42 2024-09-14T03:44:28.843 INFO:tasks.rados.rados.0.smithi110.stdout:2010: finishing write tid 2 to smithi11075562-42 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2010: finishing write tid 3 to smithi11075562-42 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2013: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2013: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2013: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1114 (ObjNum 617 snap 189 seq_num 617) dirty exists 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1445 (ObjNum 618 snap 189 seq_num 618) dirty exists 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2011: done (5 left) 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2012: done (4 left) 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2013: done (3 left) 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2016: copy_from oid 6 from oid 50 current snap is 199 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2010: finishing write tid 4 to smithi11075562-42 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1247 (ObjNum 643 snap 199 seq_num 643) dirty exists 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2010: left oid 42 (ObjNum 643 snap 199 seq_num 643) 2024-09-14T03:44:28.844 INFO:tasks.rados.rados.0.smithi110.stdout:2010: done (3 left) 2024-09-14T03:44:28.845 INFO:tasks.rados.rados.0.smithi110.stdout:2017: delete oid 21 current snap is 199 2024-09-14T03:44:28.845 INFO:tasks.rados.rados.0.smithi110.stdout:2014: finishing write tid 1 to smithi11075562-8 2024-09-14T03:44:28.846 INFO:tasks.rados.rados.0.smithi110.stdout:2017: done (3 left) 2024-09-14T03:44:28.846 INFO:tasks.rados.rados.0.smithi110.stdout:2018: rmattr oid 26 current snap is 199 2024-09-14T03:44:28.846 INFO:tasks.rados.rados.0.smithi110.stdout:2018: done (3 left) 2024-09-14T03:44:28.846 INFO:tasks.rados.rados.0.smithi110.stdout:2019: copy_from oid 46 from oid 45 current snap is 199 2024-09-14T03:44:28.846 INFO:tasks.rados.rados.0.smithi110.stdout:2020: setattr oid 23 current snap is 199 2024-09-14T03:44:28.847 INFO:tasks.rados.rados.0.smithi110.stdout:2014: finishing write tid 2 to smithi11075562-8 2024-09-14T03:44:28.847 INFO:tasks.rados.rados.0.smithi110.stdout:2014: finishing write tid 3 to smithi11075562-8 2024-09-14T03:44:28.848 INFO:tasks.rados.rados.0.smithi110.stdout:2021: snap_create 2024-09-14T03:44:28.848 INFO:tasks.rados.rados.0.smithi110.stdout:2015: finishing rollback tid 0 to smithi11075562-28 2024-09-14T03:44:28.848 INFO:tasks.rados.rados.0.smithi110.stdout:2015: finishing rollback tid 1 to smithi11075562-28 2024-09-14T03:44:28.848 INFO:tasks.rados.rados.0.smithi110.stdout:2015: finishing rollback tid 2 to smithi11075562-28 2024-09-14T03:44:28.848 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 731 (ObjNum 600 snap 183 seq_num 600) dirty exists 2024-09-14T03:44:28.849 INFO:tasks.rados.rados.0.smithi110.stdout:2014: finishing write tid 4 to smithi11075562-8 2024-09-14T03:44:28.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1235 (ObjNum 644 snap 199 seq_num 644) dirty exists 2024-09-14T03:44:28.849 INFO:tasks.rados.rados.0.smithi110.stdout:2014: left oid 8 (ObjNum 644 snap 199 seq_num 644) 2024-09-14T03:44:28.850 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 924 (ObjNum 33 snap 0 seq_num 1493618952) dirty exists 2024-09-14T03:44:28.851 INFO:tasks.rados.rados.0.smithi110.stdout:2019: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:44:28.852 INFO:tasks.rados.rados.0.smithi110.stdout:2019: finishing copy_from to smithi11075562-46 2024-09-14T03:44:28.852 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1435 (ObjNum 628 snap 195 seq_num 628) dirty exists 2024-09-14T03:44:28.860 INFO:tasks.rados.rados.0.smithi110.stdout:2016: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:44:28.865 INFO:tasks.rados.rados.0.smithi110.stdout:2016: finishing copy_from to smithi11075562-6 2024-09-14T03:44:28.865 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 937 (ObjNum 606 snap 187 seq_num 606) dirty exists 2024-09-14T03:44:29.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:28 smithi195 ceph-mon[82388]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T03:44:29.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:28 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1745861042' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:29.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:28 smithi195 ceph-mon[82388]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:29.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:28 smithi195 ceph-mon[82388]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T03:44:29.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:28 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:44:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[88262]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T03:44:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1745861042' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[88262]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[88262]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T03:44:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:44:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[86423]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T03:44:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1745861042' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[86423]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[86423]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T03:44:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:28 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:44:29.808 INFO:tasks.rados.rados.0.smithi110.stdout:2014: done (5 left) 2024-09-14T03:44:29.808 INFO:tasks.rados.rados.0.smithi110.stdout:2015: done (4 left) 2024-09-14T03:44:29.808 INFO:tasks.rados.rados.0.smithi110.stdout:2016: done (3 left) 2024-09-14T03:44:29.808 INFO:tasks.rados.rados.0.smithi110.stdout:2019: done (2 left) 2024-09-14T03:44:29.808 INFO:tasks.rados.rados.0.smithi110.stdout:2020: done (1 left) 2024-09-14T03:44:29.809 INFO:tasks.rados.rados.0.smithi110.stdout:2021: done (0 left) 2024-09-14T03:44:29.809 INFO:tasks.rados.rados.0.smithi110.stdout:2022: delete oid 24 current snap is 200 2024-09-14T03:44:29.814 INFO:tasks.rados.rados.0.smithi110.stdout:2022: done (0 left) 2024-09-14T03:44:29.814 INFO:tasks.rados.rados.0.smithi110.stdout:2023: rmattr oid 14 current snap is 200 2024-09-14T03:44:29.815 INFO:tasks.rados.rados.0.smithi110.stdout:2024: delete oid 7 current snap is 200 2024-09-14T03:44:29.817 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1118 (ObjNum 603 snap 185 seq_num 603) dirty exists 2024-09-14T03:44:29.819 INFO:tasks.rados.rados.0.smithi110.stdout:2023: done (1 left) 2024-09-14T03:44:29.819 INFO:tasks.rados.rados.0.smithi110.stdout:2024: done (0 left) 2024-09-14T03:44:29.819 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 200 2024-09-14T03:44:29.819 INFO:tasks.rados.rados.0.smithi110.stdout:2025: seq_num 648 ranges {0=458752,458752=475136,933888=327680} 2024-09-14T03:44:29.821 INFO:tasks.rados.rados.0.smithi110.stdout:2025: writing smithi11075562-4 from 0 to 458752 tid 1 2024-09-14T03:44:29.824 INFO:tasks.rados.rados.0.smithi110.stdout:2025: writing smithi11075562-4 from 458752 to 933888 tid 2 2024-09-14T03:44:29.826 INFO:tasks.rados.rados.0.smithi110.stdout:2025: writing smithi11075562-4 from 933888 to 1261568 tid 3 2024-09-14T03:44:29.827 INFO:tasks.rados.rados.0.smithi110.stdout:2026: snap_create 2024-09-14T03:44:29.827 INFO:tasks.rados.rados.0.smithi110.stdout:2025: finishing write tid 1 to smithi11075562-4 2024-09-14T03:44:29.828 INFO:tasks.rados.rados.0.smithi110.stdout:2025: finishing write tid 2 to smithi11075562-4 2024-09-14T03:44:29.828 INFO:tasks.rados.rados.0.smithi110.stdout:2025: finishing write tid 3 to smithi11075562-4 2024-09-14T03:44:29.829 INFO:tasks.rados.rados.0.smithi110.stdout:2025: finishing write tid 4 to smithi11075562-4 2024-09-14T03:44:29.829 INFO:tasks.rados.rados.0.smithi110.stdout:2025: finishing write tid 5 to smithi11075562-4 2024-09-14T03:44:29.829 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1159 (ObjNum 648 snap 200 seq_num 648) dirty exists 2024-09-14T03:44:29.829 INFO:tasks.rados.rados.0.smithi110.stdout:2025: left oid 4 (ObjNum 648 snap 200 seq_num 648) 2024-09-14T03:44:30.009 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:29 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:44:30.009 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:29 smithi195 ceph-mon[82388]: Upgrade: osd.5 is safe to restart 2024-09-14T03:44:30.010 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:29 smithi195 ceph-mon[82388]: pgmap v333: 121 pgs: 121 active+clean; 203 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 25 op/s 2024-09-14T03:44:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:29 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:44:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:29 smithi110 ceph-mon[88262]: Upgrade: osd.5 is safe to restart 2024-09-14T03:44:30.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:29 smithi110 ceph-mon[88262]: pgmap v333: 121 pgs: 121 active+clean; 203 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 25 op/s 2024-09-14T03:44:30.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:29 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:44:30.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:29 smithi110 ceph-mon[86423]: Upgrade: osd.5 is safe to restart 2024-09-14T03:44:30.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:29 smithi110 ceph-mon[86423]: pgmap v333: 121 pgs: 121 active+clean; 203 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 25 op/s 2024-09-14T03:44:30.378 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:30.814 INFO:tasks.rados.rados.0.smithi110.stdout:2025: done (1 left) 2024-09-14T03:44:30.814 INFO:tasks.rados.rados.0.smithi110.stdout:2026: done (0 left) 2024-09-14T03:44:30.814 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 201 2024-09-14T03:44:30.814 INFO:tasks.rados.rados.0.smithi110.stdout:2027: seq_num 649 ranges {925696=475136,1400832=761856,2162688=40960} 2024-09-14T03:44:30.821 INFO:tasks.rados.rados.0.smithi110.stdout:2027: writing smithi11075562-45 from 925696 to 1400832 tid 1 2024-09-14T03:44:30.826 INFO:tasks.rados.rados.0.smithi110.stdout:2027: writing smithi11075562-45 from 1400832 to 2162688 tid 2 2024-09-14T03:44:30.826 INFO:tasks.rados.rados.0.smithi110.stdout:2027: writing smithi11075562-45 from 2162688 to 2203648 tid 3 2024-09-14T03:44:30.826 INFO:tasks.rados.rados.0.smithi110.stdout:2028: snap_remove snap 198 2024-09-14T03:44:30.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:30 smithi195 ceph-mon[82388]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T03:44:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:30 smithi195 ceph-mon[82388]: Upgrade: Updating osd.5 2024-09-14T03:44:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:30 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:30 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:44:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:30 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:30 smithi195 ceph-mon[82388]: Deploying daemon osd.5 on smithi195 2024-09-14T03:44:31.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[88262]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T03:44:31.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[88262]: Upgrade: Updating osd.5 2024-09-14T03:44:31.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:31.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:44:31.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:31.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[88262]: Deploying daemon osd.5 on smithi195 2024-09-14T03:44:31.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[86423]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T03:44:31.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[86423]: Upgrade: Updating osd.5 2024-09-14T03:44:31.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:31.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:44:31.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:31.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:30 smithi110 ceph-mon[86423]: Deploying daemon osd.5 on smithi195 2024-09-14T03:44:31.815 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:44:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:44:31] "GET /metrics HTTP/1.1" 200 36248 "" "Prometheus/2.51.0" 2024-09-14T03:44:31.820 INFO:tasks.rados.rados.0.smithi110.stdout:2027: finishing write tid 1 to smithi11075562-45 2024-09-14T03:44:31.820 INFO:tasks.rados.rados.0.smithi110.stdout:2027: finishing write tid 2 to smithi11075562-45 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:2027: finishing write tid 3 to smithi11075562-45 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:2027: finishing write tid 4 to smithi11075562-45 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:2027: finishing write tid 5 to smithi11075562-45 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 1450 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:2027: left oid 45 (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:2027: done (1 left) 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:2028: done (0 left) 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:2029: copy_from oid 41 from oid 23 current snap is 201 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 201 2024-09-14T03:44:31.821 INFO:tasks.rados.rados.0.smithi110.stdout:2030: seq_num 650 ranges {114688=491520,606208=647168,1253376=155648} 2024-09-14T03:44:31.825 INFO:tasks.rados.rados.0.smithi110.stdout:2030: writing smithi11075562-32 from 114688 to 606208 tid 1 2024-09-14T03:44:31.829 INFO:tasks.rados.rados.0.smithi110.stdout:2030: writing smithi11075562-32 from 606208 to 1253376 tid 2 2024-09-14T03:44:31.830 INFO:tasks.rados.rados.0.smithi110.stdout:2030: writing smithi11075562-32 from 1253376 to 1409024 tid 3 2024-09-14T03:44:31.830 INFO:tasks.rados.rados.0.smithi110.stdout:2031: snap_remove snap 197 2024-09-14T03:44:31.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:31 smithi195 ceph-mon[82388]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T03:44:31.979 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:31 smithi195 ceph-mon[82388]: pgmap v336: 121 pgs: 121 active+clean; 207 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:44:32.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:31 smithi110 ceph-mon[88262]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T03:44:32.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:31 smithi110 ceph-mon[88262]: pgmap v336: 121 pgs: 121 active+clean; 207 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:44:32.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:31 smithi110 ceph-mon[86423]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T03:44:32.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:31 smithi110 ceph-mon[86423]: pgmap v336: 121 pgs: 121 active+clean; 207 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T03:44:32.825 INFO:tasks.rados.rados.0.smithi110.stdout:2031: done (2 left) 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2032: delete oid 19 current snap is 201 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2029: finishing copy_from racing read to smithi11075562-41 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2030: finishing write tid 1 to smithi11075562-32 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2029: finishing copy_from to smithi11075562-41 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 924 (ObjNum 129 snap 0 seq_num 65084632) dirty exists 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2030: finishing write tid 2 to smithi11075562-32 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2030: finishing write tid 3 to smithi11075562-32 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2030: finishing write tid 4 to smithi11075562-32 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2030: finishing write tid 5 to smithi11075562-32 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1239 (ObjNum 650 snap 201 seq_num 650) dirty exists 2024-09-14T03:44:32.826 INFO:tasks.rados.rados.0.smithi110.stdout:2030: left oid 32 (ObjNum 650 snap 201 seq_num 650) 2024-09-14T03:44:32.835 INFO:tasks.rados.rados.0.smithi110.stdout:2029: done (2 left) 2024-09-14T03:44:32.836 INFO:tasks.rados.rados.0.smithi110.stdout:2030: done (1 left) 2024-09-14T03:44:32.836 INFO:tasks.rados.rados.0.smithi110.stdout:2032: done (0 left) 2024-09-14T03:44:32.836 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 201 2024-09-14T03:44:32.836 INFO:tasks.rados.rados.0.smithi110.stdout:2033: seq_num 652 ranges {0=524288,524288=409600,933888=393216} 2024-09-14T03:44:32.839 INFO:tasks.rados.rados.0.smithi110.stdout:2033: writing smithi11075562-23 from 0 to 524288 tid 1 2024-09-14T03:44:32.841 INFO:tasks.rados.rados.0.smithi110.stdout:2033: writing smithi11075562-23 from 524288 to 933888 tid 2 2024-09-14T03:44:32.844 INFO:tasks.rados.rados.0.smithi110.stdout:2033: writing smithi11075562-23 from 933888 to 1327104 tid 3 2024-09-14T03:44:32.844 INFO:tasks.rados.rados.0.smithi110.stdout:2034: snap_create 2024-09-14T03:44:32.844 INFO:tasks.rados.rados.0.smithi110.stdout:2033: finishing write tid 1 to smithi11075562-23 2024-09-14T03:44:32.847 INFO:tasks.rados.rados.0.smithi110.stdout:2033: finishing write tid 2 to smithi11075562-23 2024-09-14T03:44:32.857 INFO:tasks.rados.rados.0.smithi110.stdout:2033: finishing write tid 3 to smithi11075562-23 2024-09-14T03:44:32.857 INFO:tasks.rados.rados.0.smithi110.stdout:2033: finishing write tid 4 to smithi11075562-23 2024-09-14T03:44:32.858 INFO:tasks.rados.rados.0.smithi110.stdout:2033: finishing write tid 5 to smithi11075562-23 2024-09-14T03:44:32.858 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 929 (ObjNum 652 snap 201 seq_num 652) dirty exists 2024-09-14T03:44:32.858 INFO:tasks.rados.rados.0.smithi110.stdout:2033: left oid 23 (ObjNum 652 snap 201 seq_num 652) 2024-09-14T03:44:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:32 smithi195 ceph-mon[82388]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T03:44:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:32 smithi110 ceph-mon[86423]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T03:44:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:32 smithi110 ceph-mon[88262]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T03:44:33.301 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:33 smithi195 systemd[1]: Stopping Ceph osd.5 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:44:33.708 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[41452]: 2024-09-14T03:44:33.318+0000 7f2d5193e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:44:33.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[41452]: 2024-09-14T03:44:33.318+0000 7f2d5193e640 -1 osd.5 474 *** Got signal Terminated *** 2024-09-14T03:44:33.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[41452]: 2024-09-14T03:44:33.318+0000 7f2d5193e640 -1 osd.5 474 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:44:33.720 INFO:tasks.rados.rados.0.smithi110.stdout:2033: done (1 left) 2024-09-14T03:44:33.720 INFO:tasks.rados.rados.0.smithi110.stdout:2034: done (0 left) 2024-09-14T03:44:33.720 INFO:tasks.rados.rados.0.smithi110.stdout:2035: read oid 33 snap -1 2024-09-14T03:44:33.720 INFO:tasks.rados.rados.0.smithi110.stdout:2035: expect (ObjNum 508 snap 160 seq_num 508) 2024-09-14T03:44:33.720 INFO:tasks.rados.rados.0.smithi110.stdout:2036: read oid 5 snap -1 2024-09-14T03:44:33.720 INFO:tasks.rados.rados.0.smithi110.stdout:2036: expect (ObjNum 542 snap 166 seq_num 542) 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2037: rollback oid 43 current snap is 202 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 43 to 196 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2038: read oid 23 snap -1 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2038: expect (ObjNum 652 snap 201 seq_num 652) 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2039: copy_from oid 32 from oid 18 current snap is 202 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2040: read oid 9 snap -1 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2040: expect deleted 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2041: rollback oid 34 current snap is 202 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 201 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2042: read oid 40 snap -1 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2042: expect deleted 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 202 2024-09-14T03:44:33.721 INFO:tasks.rados.rados.0.smithi110.stdout:2043: seq_num 653 ranges {0=540672,540672=696320,1236992=106496} 2024-09-14T03:44:33.723 INFO:tasks.rados.rados.0.smithi110.stdout:2043: writing smithi11075562-30 from 0 to 540672 tid 1 2024-09-14T03:44:33.728 INFO:tasks.rados.rados.0.smithi110.stdout:2043: writing smithi11075562-30 from 540672 to 1236992 tid 2 2024-09-14T03:44:33.730 INFO:tasks.rados.rados.0.smithi110.stdout:2043: writing smithi11075562-30 from 1236992 to 1343488 tid 3 2024-09-14T03:44:33.730 INFO:tasks.rados.rados.0.smithi110.stdout:2040: done (8 left) 2024-09-14T03:44:33.730 INFO:tasks.rados.rados.0.smithi110.stdout:2042: done (7 left) 2024-09-14T03:44:33.731 INFO:tasks.rados.rados.0.smithi110.stdout:2044: rollback oid 12 current snap is 202 2024-09-14T03:44:33.731 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 194 2024-09-14T03:44:33.731 INFO:tasks.rados.rados.0.smithi110.stdout:2045: snap_remove snap 190 2024-09-14T03:44:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[86423]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T03:44:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[86423]: osd.5 marked itself down and dead 2024-09-14T03:44:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[86423]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:44:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[86423]: osdmap e475: 8 total, 7 up, 8 in 2024-09-14T03:44:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[86423]: pgmap v340: 121 pgs: 19 stale+active+clean, 102 active+clean; 211 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-14T03:44:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[88262]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T03:44:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[88262]: osd.5 marked itself down and dead 2024-09-14T03:44:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[88262]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:44:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[88262]: osdmap e475: 8 total, 7 up, 8 in 2024-09-14T03:44:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:33 smithi110 ceph-mon[88262]: pgmap v340: 121 pgs: 19 stale+active+clean, 102 active+clean; 211 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-14T03:44:34.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:33 smithi195 ceph-mon[82388]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T03:44:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:33 smithi195 ceph-mon[82388]: osd.5 marked itself down and dead 2024-09-14T03:44:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:33 smithi195 ceph-mon[82388]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:44:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:33 smithi195 ceph-mon[82388]: osdmap e475: 8 total, 7 up, 8 in 2024-09-14T03:44:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:33 smithi195 ceph-mon[82388]: pgmap v340: 121 pgs: 19 stale+active+clean, 102 active+clean; 211 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-14T03:44:34.579 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:34 smithi195 podman[91343]: 2024-09-14 03:44:34.288554294 +0000 UTC m=+1.130729904 container died d7b0180ede9d9d29ea289e1502886d17043899b518d67cb74a6bb8b8f9bd11f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:44:34.726 INFO:tasks.rados.rados.0.smithi110.stdout:2035: expect (ObjNum 508 snap 160 seq_num 508) 2024-09-14T03:44:34.780 INFO:tasks.rados.rados.0.smithi110.stdout:2037: finishing rollback tid 0 to smithi11075562-43 2024-09-14T03:44:34.780 INFO:tasks.rados.rados.0.smithi110.stdout:2037: finishing rollback tid 1 to smithi11075562-43 2024-09-14T03:44:34.780 INFO:tasks.rados.rados.0.smithi110.stdout:2037: finishing rollback tid 2 to smithi11075562-43 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1138 (ObjNum 613 snap 188 seq_num 613) dirty exists 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2039: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2043: finishing write tid 1 to smithi11075562-30 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2043: finishing write tid 2 to smithi11075562-30 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2043: finishing write tid 3 to smithi11075562-30 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2039: finishing copy_from to smithi11075562-32 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1097 (ObjNum 595 snap 181 seq_num 595) dirty exists 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2043: finishing write tid 4 to smithi11075562-30 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2043: finishing write tid 5 to smithi11075562-30 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1243 (ObjNum 653 snap 202 seq_num 653) dirty exists 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2043: left oid 30 (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2035: done (8 left) 2024-09-14T03:44:34.781 INFO:tasks.rados.rados.0.smithi110.stdout:2037: done (7 left) 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:2039: done (6 left) 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:2043: done (5 left) 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:2045: done (4 left) 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:2046: rollback oid 2 current snap is 202 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 2 to 199 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:2041: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:2041: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:2041: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1162 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:44:34.782 INFO:tasks.rados.rados.0.smithi110.stdout:2038: expect (ObjNum 652 snap 201 seq_num 652) 2024-09-14T03:44:34.816 INFO:tasks.rados.rados.0.smithi110.stdout:2036: expect (ObjNum 542 snap 166 seq_num 542) 2024-09-14T03:44:34.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:34 smithi195 podman[91343]: 2024-09-14 03:44:34.760839672 +0000 UTC m=+1.603015285 container cleanup d7b0180ede9d9d29ea289e1502886d17043899b518d67cb74a6bb8b8f9bd11f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T03:44:34.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:34 smithi195 bash[91343]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5 2024-09-14T03:44:34.964 INFO:tasks.rados.rados.0.smithi110.stdout:2044: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:44:34.964 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 0 (ObjNum 129 snap 0 seq_num 66752512) dirty dne 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2046: finishing rollback tid 0 to smithi11075562-2 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2046: finishing rollback tid 1 to smithi11075562-2 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2046: finishing rollback tid 2 to smithi11075562-2 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 743 (ObjNum 455 snap 147 seq_num 455) dirty exists 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2036: done (4 left) 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2038: done (3 left) 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2041: done (2 left) 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2044: done (1 left) 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2046: done (0 left) 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 202 2024-09-14T03:44:34.965 INFO:tasks.rados.rados.0.smithi110.stdout:2047: seq_num 654 ranges {0=557056,557056=630784,1187840=172032} 2024-09-14T03:44:34.967 INFO:tasks.rados.rados.0.smithi110.stdout:2047: writing smithi11075562-9 from 0 to 557056 tid 1 2024-09-14T03:44:34.971 INFO:tasks.rados.rados.0.smithi110.stdout:2047: writing smithi11075562-9 from 557056 to 1187840 tid 2 2024-09-14T03:44:34.972 INFO:tasks.rados.rados.0.smithi110.stdout:2047: writing smithi11075562-9 from 1187840 to 1359872 tid 3 2024-09-14T03:44:34.972 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 202 2024-09-14T03:44:34.972 INFO:tasks.rados.rados.0.smithi110.stdout:2048: seq_num 655 ranges {7544832=573440,8118272=516096,8634368=286720} 2024-09-14T03:44:35.015 INFO:tasks.rados.rados.0.smithi110.stdout:2048: writing smithi11075562-39 from 7544832 to 8118272 tid 1 2024-09-14T03:44:35.018 INFO:tasks.rados.rados.0.smithi110.stdout:2048: writing smithi11075562-39 from 8118272 to 8634368 tid 2 2024-09-14T03:44:35.020 INFO:tasks.rados.rados.0.smithi110.stdout:2048: writing smithi11075562-39 from 8634368 to 8921088 tid 3 2024-09-14T03:44:35.020 INFO:tasks.rados.rados.0.smithi110.stdout:2049: snap_create 2024-09-14T03:44:35.020 INFO:tasks.rados.rados.0.smithi110.stdout:2047: finishing write tid 1 to smithi11075562-9 2024-09-14T03:44:35.020 INFO:tasks.rados.rados.0.smithi110.stdout:2047: finishing write tid 2 to smithi11075562-9 2024-09-14T03:44:35.021 INFO:tasks.rados.rados.0.smithi110.stdout:2047: finishing write tid 3 to smithi11075562-9 2024-09-14T03:44:35.021 INFO:tasks.rados.rados.0.smithi110.stdout:2047: finishing write tid 4 to smithi11075562-9 2024-09-14T03:44:35.021 INFO:tasks.rados.rados.0.smithi110.stdout:2047: finishing write tid 5 to smithi11075562-9 2024-09-14T03:44:35.021 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1439 (ObjNum 654 snap 202 seq_num 654) dirty exists 2024-09-14T03:44:35.021 INFO:tasks.rados.rados.0.smithi110.stdout:2047: left oid 9 (ObjNum 654 snap 202 seq_num 654) 2024-09-14T03:44:35.021 INFO:tasks.rados.rados.0.smithi110.stdout:2048: finishing write tid 1 to smithi11075562-39 2024-09-14T03:44:35.023 INFO:tasks.rados.rados.0.smithi110.stdout:2048: finishing write tid 2 to smithi11075562-39 2024-09-14T03:44:35.023 INFO:tasks.rados.rados.0.smithi110.stdout:2048: finishing write tid 3 to smithi11075562-39 2024-09-14T03:44:35.023 INFO:tasks.rados.rados.0.smithi110.stdout:2048: finishing write tid 4 to smithi11075562-39 2024-09-14T03:44:35.024 INFO:tasks.rados.rados.0.smithi110.stdout:2048: finishing write tid 5 to smithi11075562-39 2024-09-14T03:44:35.024 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1153 (ObjNum 655 snap 202 seq_num 655) dirty exists 2024-09-14T03:44:35.024 INFO:tasks.rados.rados.0.smithi110.stdout:2048: left oid 39 (ObjNum 655 snap 202 seq_num 655) 2024-09-14T03:44:35.369 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:35 smithi195 podman[91359]: 2024-09-14 03:44:35.037629308 +0000 UTC m=+0.746548626 container remove d7b0180ede9d9d29ea289e1502886d17043899b518d67cb74a6bb8b8f9bd11f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-14T03:44:35.633 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:35 smithi195 podman[91448]: 2024-09-14 03:44:35.271455194 +0000 UTC m=+0.016296556 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:44:35.633 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:35 smithi195 podman[91448]: 2024-09-14 03:44:35.388453363 +0000 UTC m=+0.133294764 container create a75b831fff6816cbc04421ea4a7ed22cd115c79f47048fbf170dab198f43fe1d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-14T03:44:35.725 INFO:tasks.rados.rados.0.smithi110.stdout:2047: done (2 left) 2024-09-14T03:44:35.725 INFO:tasks.rados.rados.0.smithi110.stdout:2048: done (1 left) 2024-09-14T03:44:35.725 INFO:tasks.rados.rados.0.smithi110.stdout:2049: done (0 left) 2024-09-14T03:44:35.726 INFO:tasks.rados.rados.0.smithi110.stdout:2050: delete oid 50 current snap is 203 2024-09-14T03:44:35.732 INFO:tasks.rados.rados.0.smithi110.stdout:2050: done (0 left) 2024-09-14T03:44:35.732 INFO:tasks.rados.rados.0.smithi110.stdout:2051: rmattr oid 47 current snap is 203 2024-09-14T03:44:35.732 INFO:tasks.rados.rados.0.smithi110.stdout:2052: delete oid 15 current snap is 203 2024-09-14T03:44:35.735 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1465 (ObjNum 618 snap 189 seq_num 618) dirty exists 2024-09-14T03:44:35.735 INFO:tasks.rados.rados.0.smithi110.stdout:2051: done (1 left) 2024-09-14T03:44:35.735 INFO:tasks.rados.rados.0.smithi110.stdout:2052: done (0 left) 2024-09-14T03:44:35.735 INFO:tasks.rados.rados.0.smithi110.stdout:2053: snap_remove snap 191 2024-09-14T03:44:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:35 smithi195 ceph-mon[82388]: osdmap e476: 8 total, 7 up, 8 in 2024-09-14T03:44:35.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:35 smithi195 podman[91448]: 2024-09-14 03:44:35.63246554 +0000 UTC m=+0.377306894 container init a75b831fff6816cbc04421ea4a7ed22cd115c79f47048fbf170dab198f43fe1d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-09-14T03:44:35.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:35 smithi195 podman[91448]: 2024-09-14 03:44:35.635779493 +0000 UTC m=+0.380620846 container start a75b831fff6816cbc04421ea4a7ed22cd115c79f47048fbf170dab198f43fe1d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:44:35.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:35 smithi195 podman[91448]: 2024-09-14 03:44:35.724038742 +0000 UTC m=+0.468880101 container attach a75b831fff6816cbc04421ea4a7ed22cd115c79f47048fbf170dab198f43fe1d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-14T03:44:35.960 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:35 smithi195 podman[91448]: 2024-09-14 03:44:35.827117908 +0000 UTC m=+0.571959278 container died a75b831fff6816cbc04421ea4a7ed22cd115c79f47048fbf170dab198f43fe1d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-14T03:44:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:35 smithi110 ceph-mon[88262]: osdmap e476: 8 total, 7 up, 8 in 2024-09-14T03:44:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:35 smithi110 ceph-mon[86423]: osdmap e476: 8 total, 7 up, 8 in 2024-09-14T03:44:36.731 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:36 smithi195 podman[91471]: 2024-09-14 03:44:36.478057191 +0000 UTC m=+0.644960763 container remove a75b831fff6816cbc04421ea4a7ed22cd115c79f47048fbf170dab198f43fe1d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-a278231, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-14T03:44:36.732 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:36 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-14T03:44:36.732 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:36 smithi195 systemd[1]: Stopped Ceph osd.5 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:44:36.732 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:36 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.5.service: Consumed 17.476s CPU time. 2024-09-14T03:44:36.732 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:36 smithi195 systemd[1]: Starting Ceph osd.5 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:44:36.737 INFO:tasks.rados.rados.0.smithi110.stdout:2053: done (0 left) 2024-09-14T03:44:36.737 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 203 2024-09-14T03:44:36.737 INFO:tasks.rados.rados.0.smithi110.stdout:2054: seq_num 658 ranges {1327104=622592,1949696=565248,2514944=237568} 2024-09-14T03:44:36.749 INFO:tasks.rados.rados.0.smithi110.stdout:2054: writing smithi11075562-23 from 1327104 to 1949696 tid 1 2024-09-14T03:44:36.753 INFO:tasks.rados.rados.0.smithi110.stdout:2054: writing smithi11075562-23 from 1949696 to 2514944 tid 2 2024-09-14T03:44:36.754 INFO:tasks.rados.rados.0.smithi110.stdout:2054: writing smithi11075562-23 from 2514944 to 2752512 tid 3 2024-09-14T03:44:36.755 INFO:tasks.rados.rados.0.smithi110.stdout:2055: read oid 15 snap -1 2024-09-14T03:44:36.755 INFO:tasks.rados.rados.0.smithi110.stdout:2055: expect deleted 2024-09-14T03:44:36.755 INFO:tasks.rados.rados.0.smithi110.stdout:2056: setattr oid 36 current snap is 203 2024-09-14T03:44:36.756 INFO:tasks.rados.rados.0.smithi110.stdout:2057: snap_create 2024-09-14T03:44:36.759 INFO:tasks.rados.rados.0.smithi110.stdout:2054: finishing write tid 1 to smithi11075562-23 2024-09-14T03:44:36.760 INFO:tasks.rados.rados.0.smithi110.stdout:2054: finishing write tid 2 to smithi11075562-23 2024-09-14T03:44:36.760 INFO:tasks.rados.rados.0.smithi110.stdout:2054: finishing write tid 3 to smithi11075562-23 2024-09-14T03:44:36.760 INFO:tasks.rados.rados.0.smithi110.stdout:2054: finishing write tid 4 to smithi11075562-23 2024-09-14T03:44:36.760 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1467 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-09-14T03:44:36.761 INFO:tasks.rados.rados.0.smithi110.stdout:2054: finishing write tid 5 to smithi11075562-23 2024-09-14T03:44:36.761 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 933 (ObjNum 658 snap 203 seq_num 658) dirty exists 2024-09-14T03:44:36.761 INFO:tasks.rados.rados.0.smithi110.stdout:2054: left oid 23 (ObjNum 658 snap 203 seq_num 658) 2024-09-14T03:44:37.048 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:36 smithi195 ceph-mon[82388]: osdmap e477: 8 total, 7 up, 8 in 2024-09-14T03:44:37.048 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:36 smithi195 ceph-mon[82388]: pgmap v343: 121 pgs: 4 active+undersized, 5 peering, 17 stale+active+clean, 2 active+undersized+degraded, 93 active+clean; 218 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 997 KiB/s rd, 3.2 MiB/s wr, 26 op/s; 12/354 objects degraded (3.390%) 2024-09-14T03:44:37.049 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:36 smithi195 podman[91570]: 2024-09-14 03:44:36.929440685 +0000 UTC m=+0.110941687 container create 01fda494b9c7726dfd628d17f9cf299b23074c55afc3630697d1af17c9b71079 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-14T03:44:37.049 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:36 smithi195 podman[91570]: 2024-09-14 03:44:36.834329575 +0000 UTC m=+0.015830583 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:44:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:36 smithi110 ceph-mon[86423]: osdmap e477: 8 total, 7 up, 8 in 2024-09-14T03:44:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:36 smithi110 ceph-mon[86423]: pgmap v343: 121 pgs: 4 active+undersized, 5 peering, 17 stale+active+clean, 2 active+undersized+degraded, 93 active+clean; 218 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 997 KiB/s rd, 3.2 MiB/s wr, 26 op/s; 12/354 objects degraded (3.390%) 2024-09-14T03:44:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:36 smithi110 ceph-mon[88262]: osdmap e477: 8 total, 7 up, 8 in 2024-09-14T03:44:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:36 smithi110 ceph-mon[88262]: pgmap v343: 121 pgs: 4 active+undersized, 5 peering, 17 stale+active+clean, 2 active+undersized+degraded, 93 active+clean; 218 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 997 KiB/s rd, 3.2 MiB/s wr, 26 op/s; 12/354 objects degraded (3.390%) 2024-09-14T03:44:37.314 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 podman[91570]: 2024-09-14 03:44:37.182049573 +0000 UTC m=+0.363550572 container init 01fda494b9c7726dfd628d17f9cf299b23074c55afc3630697d1af17c9b71079 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-14T03:44:37.314 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 podman[91570]: 2024-09-14 03:44:37.185813924 +0000 UTC m=+0.367314924 container start 01fda494b9c7726dfd628d17f9cf299b23074c55afc3630697d1af17c9b71079 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:44:37.314 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 podman[91570]: 2024-09-14 03:44:37.265091634 +0000 UTC m=+0.446592642 container attach 01fda494b9c7726dfd628d17f9cf299b23074c55afc3630697d1af17c9b71079 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:44:37.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:44:37.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 bash[91570]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:44:37.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:44:37.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 bash[91570]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:44:37.740 INFO:tasks.rados.rados.0.smithi110.stdout:2054: done (3 left) 2024-09-14T03:44:37.740 INFO:tasks.rados.rados.0.smithi110.stdout:2055: done (2 left) 2024-09-14T03:44:37.740 INFO:tasks.rados.rados.0.smithi110.stdout:2056: done (1 left) 2024-09-14T03:44:37.740 INFO:tasks.rados.rados.0.smithi110.stdout:2057: done (0 left) 2024-09-14T03:44:37.741 INFO:tasks.rados.rados.0.smithi110.stdout:2058: delete oid 2 current snap is 204 2024-09-14T03:44:37.747 INFO:tasks.rados.rados.0.smithi110.stdout:2058: done (0 left) 2024-09-14T03:44:37.747 INFO:tasks.rados.rados.0.smithi110.stdout:2059: copy_from oid 29 from oid 30 current snap is 204 2024-09-14T03:44:37.747 INFO:tasks.rados.rados.0.smithi110.stdout:2060: read oid 1 snap 199 2024-09-14T03:44:37.747 INFO:tasks.rados.rados.0.smithi110.stdout:2060: expect (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:44:37.747 INFO:tasks.rados.rados.0.smithi110.stdout:2061: rollback oid 17 current snap is 204 2024-09-14T03:44:37.747 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 200 2024-09-14T03:44:37.747 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 204 2024-09-14T03:44:37.747 INFO:tasks.rados.rados.0.smithi110.stdout:2062: seq_num 660 ranges {876544=663552,1540096=737280,2277376=65536} 2024-09-14T03:44:37.757 INFO:tasks.rados.rados.0.smithi110.stdout:2062: writing smithi11075562-49 from 876544 to 1540096 tid 1 2024-09-14T03:44:37.760 INFO:tasks.rados.rados.0.smithi110.stdout:2062: writing smithi11075562-49 from 1540096 to 2277376 tid 2 2024-09-14T03:44:37.761 INFO:tasks.rados.rados.0.smithi110.stdout:2062: writing smithi11075562-49 from 2277376 to 2342912 tid 3 2024-09-14T03:44:37.761 INFO:tasks.rados.rados.0.smithi110.stdout:2063: setattr oid 5 current snap is 204 2024-09-14T03:44:37.761 INFO:tasks.rados.rados.0.smithi110.stdout:2061: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:44:37.761 INFO:tasks.rados.rados.0.smithi110.stdout:2061: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:44:37.761 INFO:tasks.rados.rados.0.smithi110.stdout:2061: finishing rollback tid 2 to smithi11075562-17 2024-09-14T03:44:37.762 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1093 (ObjNum 551 snap 167 seq_num 551) dirty exists 2024-09-14T03:44:37.762 INFO:tasks.rados.rados.0.smithi110.stdout:2060: expect (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:44:37.825 INFO:tasks.rados.rados.0.smithi110.stdout:2059: finishing copy_from racing read to smithi11075562-29 2024-09-14T03:44:37.825 INFO:tasks.rados.rados.0.smithi110.stdout:2059: finishing copy_from to smithi11075562-29 2024-09-14T03:44:37.825 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1243 (ObjNum 653 snap 202 seq_num 653) dirty exists 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2062: finishing write tid 1 to smithi11075562-49 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2062: finishing write tid 2 to smithi11075562-49 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2062: finishing write tid 3 to smithi11075562-49 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2062: finishing write tid 4 to smithi11075562-49 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2062: finishing write tid 5 to smithi11075562-49 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1256 (ObjNum 660 snap 204 seq_num 660) dirty exists 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2062: left oid 49 (ObjNum 660 snap 204 seq_num 660) 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2059: done (4 left) 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2060: done (3 left) 2024-09-14T03:44:37.826 INFO:tasks.rados.rados.0.smithi110.stdout:2061: done (2 left) 2024-09-14T03:44:37.827 INFO:tasks.rados.rados.0.smithi110.stdout:2062: done (1 left) 2024-09-14T03:44:37.827 INFO:tasks.rados.rados.0.smithi110.stdout:2064: read oid 41 snap 195 2024-09-14T03:44:37.827 INFO:tasks.rados.rados.0.smithi110.stdout:2064: expect (ObjNum 426 snap 136 seq_num 426) 2024-09-14T03:44:37.827 INFO:tasks.rados.rados.0.smithi110.stdout:2065: read oid 39 snap 196 2024-09-14T03:44:37.827 INFO:tasks.rados.rados.0.smithi110.stdout:2065: expect (ObjNum 617 snap 189 seq_num 617) 2024-09-14T03:44:37.827 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 204 2024-09-14T03:44:37.827 INFO:tasks.rados.rados.0.smithi110.stdout:2066: seq_num 661 ranges {0=679936,679936=622592,1302528=180224} 2024-09-14T03:44:37.828 INFO:tasks.rados.rados.0.smithi110.stdout:2066: writing smithi11075562-19 from 0 to 679936 tid 1 2024-09-14T03:44:37.832 INFO:tasks.rados.rados.0.smithi110.stdout:2066: writing smithi11075562-19 from 679936 to 1302528 tid 2 2024-09-14T03:44:37.833 INFO:tasks.rados.rados.0.smithi110.stdout:2066: writing smithi11075562-19 from 1302528 to 1482752 tid 3 2024-09-14T03:44:37.833 INFO:tasks.rados.rados.0.smithi110.stdout:2067: snap_remove snap 200 2024-09-14T03:44:38.091 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:37 smithi195 ceph-mon[82388]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:44:38.091 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:37 smithi195 ceph-mon[82388]: Health check failed: Degraded data redundancy: 12/354 objects degraded (3.390%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:38.091 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:37 smithi195 ceph-mon[82388]: osdmap e478: 8 total, 7 up, 8 in 2024-09-14T03:44:38.091 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T03:44:38.092 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 bash[91570]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T03:44:38.092 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:44:38.092 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:37 smithi195 bash[91570]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:44:38.092 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:44:38.092 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 bash[91570]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:44:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:37 smithi110 ceph-mon[88262]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:44:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:37 smithi110 ceph-mon[88262]: Health check failed: Degraded data redundancy: 12/354 objects degraded (3.390%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:37 smithi110 ceph-mon[88262]: osdmap e478: 8 total, 7 up, 8 in 2024-09-14T03:44:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:37 smithi110 ceph-mon[86423]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:44:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:37 smithi110 ceph-mon[86423]: Health check failed: Degraded data redundancy: 12/354 objects degraded (3.390%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:37 smithi110 ceph-mon[86423]: osdmap e478: 8 total, 7 up, 8 in 2024-09-14T03:44:38.445 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:44:38.445 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 bash[91570]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:44:38.445 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-14T03:44:38.445 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 bash[91570]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-14T03:44:38.446 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T03:44:38.446 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 bash[91570]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T03:44:38.446 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate[91584]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-14T03:44:38.446 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 bash[91570]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-14T03:44:38.446 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 podman[91570]: 2024-09-14 03:44:38.117783976 +0000 UTC m=+1.299284977 container died 01fda494b9c7726dfd628d17f9cf299b23074c55afc3630697d1af17c9b71079 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate, RELEASE=main-a278231, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T03:44:38.745 INFO:tasks.rados.rados.0.smithi110.stdout:2067: done (4 left) 2024-09-14T03:44:38.745 INFO:tasks.rados.rados.0.smithi110.stdout:2068: snap_remove snap 203 2024-09-14T03:44:39.083 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:38 smithi195 ceph-mon[82388]: osdmap e479: 8 total, 7 up, 8 in 2024-09-14T03:44:39.083 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:38 smithi195 ceph-mon[82388]: pgmap v346: 121 pgs: 1 active+clean+snaptrim, 33 active+undersized, 5 peering, 12 active+undersized+degraded, 70 active+clean; 219 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.4 MiB/s wr, 13 op/s; 52/363 objects degraded (14.325%) 2024-09-14T03:44:39.083 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 podman[91724]: 2024-09-14 03:44:38.735749854 +0000 UTC m=+0.611746008 container remove 01fda494b9c7726dfd628d17f9cf299b23074c55afc3630697d1af17c9b71079 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-a278231, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:44:39.083 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 podman[91783]: 2024-09-14 03:44:38.847526079 +0000 UTC m=+0.016081295 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:44:39.084 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:38 smithi195 podman[91783]: 2024-09-14 03:44:38.961882013 +0000 UTC m=+0.130437213 container create 8fe7f08d58820a8a5a84f86c703180db19a805ebd5494f12a327d320833d525c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:44:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:38 smithi110 ceph-mon[88262]: osdmap e479: 8 total, 7 up, 8 in 2024-09-14T03:44:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:38 smithi110 ceph-mon[88262]: pgmap v346: 121 pgs: 1 active+clean+snaptrim, 33 active+undersized, 5 peering, 12 active+undersized+degraded, 70 active+clean; 219 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.4 MiB/s wr, 13 op/s; 52/363 objects degraded (14.325%) 2024-09-14T03:44:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:38 smithi110 ceph-mon[86423]: osdmap e479: 8 total, 7 up, 8 in 2024-09-14T03:44:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:38 smithi110 ceph-mon[86423]: pgmap v346: 121 pgs: 1 active+clean+snaptrim, 33 active+undersized, 5 peering, 12 active+undersized+degraded, 70 active+clean; 219 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.4 MiB/s wr, 13 op/s; 52/363 objects degraded (14.325%) 2024-09-14T03:44:39.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:39 smithi195 podman[91783]: 2024-09-14 03:44:39.214091552 +0000 UTC m=+0.382646755 container init 8fe7f08d58820a8a5a84f86c703180db19a805ebd5494f12a327d320833d525c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-14T03:44:39.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:39 smithi195 podman[91783]: 2024-09-14 03:44:39.217312907 +0000 UTC m=+0.385868107 container start 8fe7f08d58820a8a5a84f86c703180db19a805ebd5494f12a327d320833d525c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:44:39.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:39 smithi195 ceph-osd[91802]: -- 172.21.15.195:0/1893346476 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5601e581ba40 con 0x5601e57f9c00 2024-09-14T03:44:39.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:39 smithi195 bash[91783]: 8fe7f08d58820a8a5a84f86c703180db19a805ebd5494f12a327d320833d525c 2024-09-14T03:44:39.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:39 smithi195 systemd[1]: Started Ceph osd.5 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:44:39.745 INFO:tasks.rados.rados.0.smithi110.stdout:2068: done (4 left) 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:2069: copy_from oid 17 from oid 40 current snap is 204 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1469 (ObjNum 542 snap 166 seq_num 542) dirty exists 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:2066: finishing write tid 1 to smithi11075562-19 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:2066: finishing write tid 2 to smithi11075562-19 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:2066: finishing write tid 3 to smithi11075562-19 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:2066: finishing write tid 4 to smithi11075562-19 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:2066: finishing write tid 5 to smithi11075562-19 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 1124 (ObjNum 661 snap 204 seq_num 661) dirty exists 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:2066: left oid 19 (ObjNum 661 snap 204 seq_num 661) 2024-09-14T03:44:39.746 INFO:tasks.rados.rados.0.smithi110.stdout:2064: expect (ObjNum 426 snap 136 seq_num 426) 2024-09-14T03:44:40.029 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:39 smithi195 ceph-mon[82388]: osdmap e480: 8 total, 7 up, 8 in 2024-09-14T03:44:40.029 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:39 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:40.029 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:39 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:40.029 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:39 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:39 smithi110 ceph-mon[88262]: osdmap e480: 8 total, 7 up, 8 in 2024-09-14T03:44:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:39 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:39 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:39 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:39 smithi110 ceph-mon[86423]: osdmap e480: 8 total, 7 up, 8 in 2024-09-14T03:44:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:39 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:39 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:39 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:40.401 INFO:tasks.rados.rados.0.smithi110.stdout:2065: expect (ObjNum 617 snap 189 seq_num 617) 2024-09-14T03:44:40.458 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:40.894 INFO:tasks.rados.rados.0.smithi110.stdout:2069: finishing copy_from to smithi11075562-17 2024-09-14T03:44:40.894 INFO:tasks.rados.rados.0.smithi110.stdout:2069: got expected ENOENT (src dne) 2024-09-14T03:44:40.894 INFO:tasks.rados.rados.0.smithi110.stdout:2069: finishing copy_from racing read to smithi11075562-17 2024-09-14T03:44:40.894 INFO:tasks.rados.rados.0.smithi110.stdout:2063: done (4 left) 2024-09-14T03:44:40.894 INFO:tasks.rados.rados.0.smithi110.stdout:2064: done (3 left) 2024-09-14T03:44:40.894 INFO:tasks.rados.rados.0.smithi110.stdout:2065: done (2 left) 2024-09-14T03:44:40.895 INFO:tasks.rados.rados.0.smithi110.stdout:2066: done (1 left) 2024-09-14T03:44:40.895 INFO:tasks.rados.rados.0.smithi110.stdout:2069: done (0 left) 2024-09-14T03:44:40.895 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 204 2024-09-14T03:44:40.895 INFO:tasks.rados.rados.0.smithi110.stdout:2070: seq_num 662 ranges {3702784=696320,4399104=557056,4956160=245760} 2024-09-14T03:44:40.917 INFO:tasks.rados.rados.0.smithi110.stdout:2070: writing smithi11075562-5 from 3702784 to 4399104 tid 1 2024-09-14T03:44:40.919 INFO:tasks.rados.rados.0.smithi110.stdout:2070: writing smithi11075562-5 from 4399104 to 4956160 tid 2 2024-09-14T03:44:40.921 INFO:tasks.rados.rados.0.smithi110.stdout:2070: writing smithi11075562-5 from 4956160 to 5201920 tid 3 2024-09-14T03:44:40.921 INFO:tasks.rados.rados.0.smithi110.stdout:2071: read oid 27 snap -1 2024-09-14T03:44:40.921 INFO:tasks.rados.rados.0.smithi110.stdout:2071: expect (ObjNum 631 snap 195 seq_num 631) 2024-09-14T03:44:40.921 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 204 2024-09-14T03:44:40.921 INFO:tasks.rados.rados.0.smithi110.stdout:2072: seq_num 663 ranges {2727936=712704,3440640=442368,3883008=360448} 2024-09-14T03:44:40.941 INFO:tasks.rados.rados.0.smithi110.stdout:2072: writing smithi11075562-20 from 2727936 to 3440640 tid 1 2024-09-14T03:44:40.943 INFO:tasks.rados.rados.0.smithi110.stdout:2072: writing smithi11075562-20 from 3440640 to 3883008 tid 2 2024-09-14T03:44:40.946 INFO:tasks.rados.rados.0.smithi110.stdout:2072: writing smithi11075562-20 from 3883008 to 4243456 tid 3 2024-09-14T03:44:40.946 INFO:tasks.rados.rados.0.smithi110.stdout:2073: read oid 31 snap -1 2024-09-14T03:44:40.946 INFO:tasks.rados.rados.0.smithi110.stdout:2073: expect (ObjNum 640 snap 199 seq_num 640) 2024-09-14T03:44:40.946 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 204 2024-09-14T03:44:40.946 INFO:tasks.rados.rados.0.smithi110.stdout:2070: finishing write tid 1 to smithi11075562-5 2024-09-14T03:44:40.946 INFO:tasks.rados.rados.0.smithi110.stdout:2074: seq_num 664 ranges {1630208=729088,2359296=729088,3088384=73728} 2024-09-14T03:44:40.958 INFO:tasks.rados.rados.0.smithi110.stdout:2074: writing smithi11075562-38 from 1630208 to 2359296 tid 1 2024-09-14T03:44:40.963 INFO:tasks.rados.rados.0.smithi110.stdout:2074: writing smithi11075562-38 from 2359296 to 3088384 tid 2 2024-09-14T03:44:40.963 INFO:tasks.rados.rados.0.smithi110.stdout:2074: writing smithi11075562-38 from 3088384 to 3162112 tid 3 2024-09-14T03:44:40.963 INFO:tasks.rados.rados.0.smithi110.stdout:2075: rollback oid 16 current snap is 204 2024-09-14T03:44:40.964 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 196 2024-09-14T03:44:40.964 INFO:tasks.rados.rados.0.smithi110.stdout:2070: finishing write tid 2 to smithi11075562-5 2024-09-14T03:44:40.964 INFO:tasks.rados.rados.0.smithi110.stdout:2070: finishing write tid 3 to smithi11075562-5 2024-09-14T03:44:40.964 INFO:tasks.rados.rados.0.smithi110.stdout:2070: finishing write tid 4 to smithi11075562-5 2024-09-14T03:44:40.964 INFO:tasks.rados.rados.0.smithi110.stdout:2070: finishing write tid 5 to smithi11075562-5 2024-09-14T03:44:40.965 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1473 (ObjNum 662 snap 204 seq_num 662) dirty exists 2024-09-14T03:44:40.965 INFO:tasks.rados.rados.0.smithi110.stdout:2070: left oid 5 (ObjNum 662 snap 204 seq_num 662) 2024-09-14T03:44:40.965 INFO:tasks.rados.rados.0.smithi110.stdout:2072: finishing write tid 1 to smithi11075562-20 2024-09-14T03:44:40.965 INFO:tasks.rados.rados.0.smithi110.stdout:2072: finishing write tid 2 to smithi11075562-20 2024-09-14T03:44:40.965 INFO:tasks.rados.rados.0.smithi110.stdout:2071: expect (ObjNum 631 snap 195 seq_num 631) 2024-09-14T03:44:41.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:40 smithi195 ceph-mon[82388]: osdmap e481: 8 total, 7 up, 8 in 2024-09-14T03:44:41.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:40 smithi195 ceph-mon[82388]: pgmap v349: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 220 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1016 KiB/s wr, 9 op/s; 54/375 objects degraded (14.400%) 2024-09-14T03:44:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:40 smithi110 ceph-mon[88262]: osdmap e481: 8 total, 7 up, 8 in 2024-09-14T03:44:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:40 smithi110 ceph-mon[88262]: pgmap v349: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 220 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1016 KiB/s wr, 9 op/s; 54/375 objects degraded (14.400%) 2024-09-14T03:44:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:40 smithi110 ceph-mon[86423]: osdmap e481: 8 total, 7 up, 8 in 2024-09-14T03:44:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:40 smithi110 ceph-mon[86423]: pgmap v349: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 220 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1016 KiB/s wr, 9 op/s; 54/375 objects degraded (14.400%) 2024-09-14T03:44:41.458 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:41 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:44:41.111+0000 7f0e285f1640 -1 Falling back to public interface 2024-09-14T03:44:41.653 INFO:tasks.rados.rados.0.smithi110.stdout:2072: finishing write tid 3 to smithi11075562-20 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2072: finishing write tid 4 to smithi11075562-20 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2072: finishing write tid 5 to smithi11075562-20 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1443 (ObjNum 663 snap 204 seq_num 663) dirty exists 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2072: left oid 20 (ObjNum 663 snap 204 seq_num 663) 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2074: finishing write tid 1 to smithi11075562-38 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2070: done (5 left) 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2071: done (4 left) 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2072: done (3 left) 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2076: delete oid 10 current snap is 204 2024-09-14T03:44:41.654 INFO:tasks.rados.rados.0.smithi110.stdout:2074: finishing write tid 2 to smithi11075562-38 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:2074: finishing write tid 3 to smithi11075562-38 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:2074: finishing write tid 4 to smithi11075562-38 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:2075: finishing rollback tid 0 to smithi11075562-16 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:2075: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:2075: finishing rollback tid 2 to smithi11075562-16 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 1128 (ObjNum 613 snap 188 seq_num 613) dirty exists 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:2074: finishing write tid 5 to smithi11075562-38 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1247 (ObjNum 664 snap 204 seq_num 664) dirty exists 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:2074: left oid 38 (ObjNum 664 snap 204 seq_num 664) 2024-09-14T03:44:41.655 INFO:tasks.rados.rados.0.smithi110.stdout:2073: expect (ObjNum 640 snap 199 seq_num 640) 2024-09-14T03:44:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:41 smithi110 ceph-mon[88262]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:44:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:44:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:44:41] "GET /metrics HTTP/1.1" 200 36248 "" "Prometheus/2.51.0" 2024-09-14T03:44:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:41 smithi110 ceph-mon[86423]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:44:41.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:41 smithi195 ceph-mon[82388]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:44:42.340 INFO:tasks.rados.rados.0.smithi110.stdout:2073: done (3 left) 2024-09-14T03:44:42.340 INFO:tasks.rados.rados.0.smithi110.stdout:2074: done (2 left) 2024-09-14T03:44:42.340 INFO:tasks.rados.rados.0.smithi110.stdout:2075: done (1 left) 2024-09-14T03:44:42.340 INFO:tasks.rados.rados.0.smithi110.stdout:2076: done (0 left) 2024-09-14T03:44:42.340 INFO:tasks.rados.rados.0.smithi110.stdout:2077: snap_remove snap 194 2024-09-14T03:44:43.030 INFO:tasks.rados.rados.0.smithi110.stdout:2077: done (0 left) 2024-09-14T03:44:43.030 INFO:tasks.rados.rados.0.smithi110.stdout:2078: snap_create 2024-09-14T03:44:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:42 smithi110 ceph-mon[86423]: pgmap v350: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 220 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.6 MiB/s wr, 19 op/s; 57/387 objects degraded (14.729%) 2024-09-14T03:44:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:42 smithi110 ceph-mon[88262]: pgmap v350: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 220 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.6 MiB/s wr, 19 op/s; 57/387 objects degraded (14.729%) 2024-09-14T03:44:43.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:43.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:43.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:42 smithi195 ceph-mon[82388]: pgmap v350: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 220 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.6 MiB/s wr, 19 op/s; 57/387 objects degraded (14.729%) 2024-09-14T03:44:43.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:43.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:43.720 INFO:tasks.rados.rados.0.smithi110.stdout:2078: done (0 left) 2024-09-14T03:44:43.720 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 205 2024-09-14T03:44:43.720 INFO:tasks.rados.rados.0.smithi110.stdout:2079: seq_num 666 ranges {0=761856,761856=491520,1253376=311296} 2024-09-14T03:44:43.724 INFO:tasks.rados.rados.0.smithi110.stdout:2079: writing smithi11075562-10 from 0 to 761856 tid 1 2024-09-14T03:44:43.727 INFO:tasks.rados.rados.0.smithi110.stdout:2079: writing smithi11075562-10 from 761856 to 1253376 tid 2 2024-09-14T03:44:43.729 INFO:tasks.rados.rados.0.smithi110.stdout:2079: writing smithi11075562-10 from 1253376 to 1564672 tid 3 2024-09-14T03:44:43.730 INFO:tasks.rados.rados.0.smithi110.stdout:2080: snap_create 2024-09-14T03:44:43.731 INFO:tasks.rados.rados.0.smithi110.stdout:2079: finishing write tid 1 to smithi11075562-10 2024-09-14T03:44:43.733 INFO:tasks.rados.rados.0.smithi110.stdout:2079: finishing write tid 2 to smithi11075562-10 2024-09-14T03:44:43.733 INFO:tasks.rados.rados.0.smithi110.stdout:2079: finishing write tid 3 to smithi11075562-10 2024-09-14T03:44:43.733 INFO:tasks.rados.rados.0.smithi110.stdout:2079: finishing write tid 4 to smithi11075562-10 2024-09-14T03:44:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:2079: finishing write tid 5 to smithi11075562-10 2024-09-14T03:44:43.734 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 938 (ObjNum 666 snap 205 seq_num 666) dirty exists 2024-09-14T03:44:43.735 INFO:tasks.rados.rados.0.smithi110.stdout:2079: left oid 10 (ObjNum 666 snap 205 seq_num 666) 2024-09-14T03:44:44.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:43 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 57/387 objects degraded (14.729%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:44.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:43 smithi195 ceph-mon[82388]: osdmap e482: 8 total, 7 up, 8 in 2024-09-14T03:44:44.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:44.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:44.011 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:43 smithi195 ceph-mon[82388]: osdmap e483: 8 total, 7 up, 8 in 2024-09-14T03:44:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 57/387 objects degraded (14.729%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[88262]: osdmap e482: 8 total, 7 up, 8 in 2024-09-14T03:44:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[88262]: osdmap e483: 8 total, 7 up, 8 in 2024-09-14T03:44:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 57/387 objects degraded (14.729%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[86423]: osdmap e482: 8 total, 7 up, 8 in 2024-09-14T03:44:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:43 smithi110 ceph-mon[86423]: osdmap e483: 8 total, 7 up, 8 in 2024-09-14T03:44:44.726 INFO:tasks.rados.rados.0.smithi110.stdout:2079: done (1 left) 2024-09-14T03:44:44.726 INFO:tasks.rados.rados.0.smithi110.stdout:2080: done (0 left) 2024-09-14T03:44:44.726 INFO:tasks.rados.rados.0.smithi110.stdout:2081: rollback oid 27 current snap is 206 2024-09-14T03:44:44.727 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 193 2024-09-14T03:44:44.727 INFO:tasks.rados.rados.0.smithi110.stdout:2082: delete oid 39 current snap is 206 2024-09-14T03:44:44.732 INFO:tasks.rados.rados.0.smithi110.stdout:2081: finishing rollback tid 0 to smithi11075562-27 2024-09-14T03:44:44.733 INFO:tasks.rados.rados.0.smithi110.stdout:2082: done (1 left) 2024-09-14T03:44:44.733 INFO:tasks.rados.rados.0.smithi110.stdout:2083: rollback oid 5 current snap is 206 2024-09-14T03:44:44.734 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 5 to 202 2024-09-14T03:44:44.734 INFO:tasks.rados.rados.0.smithi110.stdout:2081: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:44:44.734 INFO:tasks.rados.rados.0.smithi110.stdout:2084: copy_from oid 6 from oid 24 current snap is 206 2024-09-14T03:44:44.734 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 206 2024-09-14T03:44:44.734 INFO:tasks.rados.rados.0.smithi110.stdout:2085: seq_num 668 ranges {2039808=794624,2834432=663552,3497984=139264} 2024-09-14T03:44:44.749 INFO:tasks.rados.rados.0.smithi110.stdout:2085: writing smithi11075562-17 from 2039808 to 2834432 tid 1 2024-09-14T03:44:44.753 INFO:tasks.rados.rados.0.smithi110.stdout:2085: writing smithi11075562-17 from 2834432 to 3497984 tid 2 2024-09-14T03:44:44.754 INFO:tasks.rados.rados.0.smithi110.stdout:2085: writing smithi11075562-17 from 3497984 to 3637248 tid 3 2024-09-14T03:44:44.754 INFO:tasks.rados.rados.0.smithi110.stdout:2086: snap_create 2024-09-14T03:44:44.754 INFO:tasks.rados.rados.0.smithi110.stdout:2081: finishing rollback tid 2 to smithi11075562-27 2024-09-14T03:44:44.754 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1200 (ObjNum 623 snap 190 seq_num 623) dirty exists 2024-09-14T03:44:44.754 INFO:tasks.rados.rados.0.smithi110.stdout:2084: finishing copy_from to smithi11075562-6 2024-09-14T03:44:44.754 INFO:tasks.rados.rados.0.smithi110.stdout:2084: got expected ENOENT (src dne) 2024-09-14T03:44:44.754 INFO:tasks.rados.rados.0.smithi110.stdout:2084: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:44:44.754 INFO:tasks.rados.rados.0.smithi110.stdout:2083: finishing rollback tid 0 to smithi11075562-5 2024-09-14T03:44:44.755 INFO:tasks.rados.rados.0.smithi110.stdout:2083: finishing rollback tid 1 to smithi11075562-5 2024-09-14T03:44:44.755 INFO:tasks.rados.rados.0.smithi110.stdout:2083: finishing rollback tid 2 to smithi11075562-5 2024-09-14T03:44:44.755 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1477 (ObjNum 542 snap 166 seq_num 542) dirty exists 2024-09-14T03:44:44.755 INFO:tasks.rados.rados.0.smithi110.stdout:2085: finishing write tid 1 to smithi11075562-17 2024-09-14T03:44:44.757 INFO:tasks.rados.rados.0.smithi110.stdout:2085: finishing write tid 2 to smithi11075562-17 2024-09-14T03:44:44.757 INFO:tasks.rados.rados.0.smithi110.stdout:2085: finishing write tid 3 to smithi11075562-17 2024-09-14T03:44:44.757 INFO:tasks.rados.rados.0.smithi110.stdout:2085: finishing write tid 4 to smithi11075562-17 2024-09-14T03:44:44.758 INFO:tasks.rados.rados.0.smithi110.stdout:2085: finishing write tid 5 to smithi11075562-17 2024-09-14T03:44:44.758 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1097 (ObjNum 668 snap 206 seq_num 668) dirty exists 2024-09-14T03:44:44.759 INFO:tasks.rados.rados.0.smithi110.stdout:2085: left oid 17 (ObjNum 668 snap 206 seq_num 668) 2024-09-14T03:44:44.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:44 smithi195 ceph-mon[82388]: pgmap v353: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 58/399 objects degraded (14.536%) 2024-09-14T03:44:44.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:44 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:44.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:44 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:44.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:44 smithi195 ceph-mon[82388]: osdmap e484: 8 total, 7 up, 8 in 2024-09-14T03:44:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:44 smithi110 ceph-mon[88262]: pgmap v353: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 58/399 objects degraded (14.536%) 2024-09-14T03:44:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:44 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:44 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:44 smithi110 ceph-mon[88262]: osdmap e484: 8 total, 7 up, 8 in 2024-09-14T03:44:45.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:44 smithi110 ceph-mon[86423]: pgmap v353: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 224 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 58/399 objects degraded (14.536%) 2024-09-14T03:44:45.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:44 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:45.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:44 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:45.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:44 smithi110 ceph-mon[86423]: osdmap e484: 8 total, 7 up, 8 in 2024-09-14T03:44:45.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:44:45.249+0000 7f0e285f1640 -1 osd.5 474 log_to_monitors true 2024-09-14T03:44:45.729 INFO:tasks.rados.rados.0.smithi110.stdout:2081: done (4 left) 2024-09-14T03:44:45.729 INFO:tasks.rados.rados.0.smithi110.stdout:2083: done (3 left) 2024-09-14T03:44:45.729 INFO:tasks.rados.rados.0.smithi110.stdout:2084: done (2 left) 2024-09-14T03:44:45.729 INFO:tasks.rados.rados.0.smithi110.stdout:2085: done (1 left) 2024-09-14T03:44:45.729 INFO:tasks.rados.rados.0.smithi110.stdout:2086: done (0 left) 2024-09-14T03:44:45.730 INFO:tasks.rados.rados.0.smithi110.stdout:2087: snap_remove snap 201 2024-09-14T03:44:45.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:44:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:44:45.747+0000 7f0e2039b640 -1 osd.5 474 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:44:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:45 smithi195 ceph-mon[82388]: from='osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:44:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:45 smithi195 ceph-mon[82388]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:44:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:45 smithi195 ceph-mon[82388]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:44:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:45 smithi195 ceph-mon[82388]: from='osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:45 smithi195 ceph-mon[82388]: osdmap e485: 8 total, 7 up, 8 in 2024-09-14T03:44:45.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:45 smithi195 ceph-mon[82388]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[88262]: from='osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:44:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[88262]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:44:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[88262]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:44:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[88262]: from='osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:46.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[88262]: osdmap e485: 8 total, 7 up, 8 in 2024-09-14T03:44:46.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[88262]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[86423]: from='osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:44:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[86423]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:44:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[86423]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:44:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[86423]: from='osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[86423]: osdmap e485: 8 total, 7 up, 8 in 2024-09-14T03:44:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:45 smithi110 ceph-mon[86423]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:44:46.732 INFO:tasks.rados.rados.0.smithi110.stdout:2087: done (0 left) 2024-09-14T03:44:46.732 INFO:tasks.rados.rados.0.smithi110.stdout:2088: read oid 42 snap 204 2024-09-14T03:44:46.732 INFO:tasks.rados.rados.0.smithi110.stdout:2088: expect (ObjNum 643 snap 199 seq_num 643) 2024-09-14T03:44:46.732 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 207 2024-09-14T03:44:46.732 INFO:tasks.rados.rados.0.smithi110.stdout:2089: seq_num 669 ranges {1318912=409600,1728512=548864,2277376=655360} 2024-09-14T03:44:46.743 INFO:tasks.rados.rados.0.smithi110.stdout:2089: writing smithi11075562-33 from 1318912 to 1728512 tid 1 2024-09-14T03:44:46.747 INFO:tasks.rados.rados.0.smithi110.stdout:2089: writing smithi11075562-33 from 1728512 to 2277376 tid 2 2024-09-14T03:44:46.752 INFO:tasks.rados.rados.0.smithi110.stdout:2089: writing smithi11075562-33 from 2277376 to 2932736 tid 3 2024-09-14T03:44:46.752 INFO:tasks.rados.rados.0.smithi110.stdout:2090: delete oid 6 current snap is 207 2024-09-14T03:44:46.752 INFO:tasks.rados.rados.0.smithi110.stdout:2089: finishing write tid 1 to smithi11075562-33 2024-09-14T03:44:46.753 INFO:tasks.rados.rados.0.smithi110.stdout:2089: finishing write tid 2 to smithi11075562-33 2024-09-14T03:44:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:2089: finishing write tid 3 to smithi11075562-33 2024-09-14T03:44:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:2090: done (2 left) 2024-09-14T03:44:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:2091: read oid 24 snap -1 2024-09-14T03:44:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:2091: expect deleted 2024-09-14T03:44:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:2092: read oid 39 snap -1 2024-09-14T03:44:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:2092: expect deleted 2024-09-14T03:44:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:2093: snap_create 2024-09-14T03:44:46.761 INFO:tasks.rados.rados.0.smithi110.stdout:2089: finishing write tid 4 to smithi11075562-33 2024-09-14T03:44:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:2089: finishing write tid 5 to smithi11075562-33 2024-09-14T03:44:46.762 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1447 (ObjNum 669 snap 207 seq_num 669) dirty exists 2024-09-14T03:44:46.763 INFO:tasks.rados.rados.0.smithi110.stdout:2089: left oid 33 (ObjNum 669 snap 207 seq_num 669) 2024-09-14T03:44:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[88262]: pgmap v356: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 226 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 1.4 MiB/s wr, 9 op/s; 59/402 objects degraded (14.677%) 2024-09-14T03:44:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[88262]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:44:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[88262]: osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993] boot 2024-09-14T03:44:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[88262]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T03:44:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:44:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[86423]: pgmap v356: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 226 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 1.4 MiB/s wr, 9 op/s; 59/402 objects degraded (14.677%) 2024-09-14T03:44:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[86423]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:44:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[86423]: osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993] boot 2024-09-14T03:44:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[86423]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T03:44:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:46 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:44:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:46 smithi195 ceph-mon[82388]: pgmap v356: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 226 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 1.4 MiB/s wr, 9 op/s; 59/402 objects degraded (14.677%) 2024-09-14T03:44:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:46 smithi195 ceph-mon[82388]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:44:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:46 smithi195 ceph-mon[82388]: osd.5 [v2:172.21.15.195:6808/3630816993,v1:172.21.15.195:6809/3630816993] boot 2024-09-14T03:44:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:46 smithi195 ceph-mon[82388]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T03:44:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:46 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:44:47.732 INFO:tasks.rados.rados.0.smithi110.stdout:2089: done (4 left) 2024-09-14T03:44:47.732 INFO:tasks.rados.rados.0.smithi110.stdout:2092: done (3 left) 2024-09-14T03:44:47.732 INFO:tasks.rados.rados.0.smithi110.stdout:2093: done (2 left) 2024-09-14T03:44:47.732 INFO:tasks.rados.rados.0.smithi110.stdout:2094: delete oid 4 current snap is 208 2024-09-14T03:44:47.759 INFO:tasks.rados.rados.0.smithi110.stdout:2091: done (2 left) 2024-09-14T03:44:47.759 INFO:tasks.rados.rados.0.smithi110.stdout:2094: done (1 left) 2024-09-14T03:44:47.759 INFO:tasks.rados.rados.0.smithi110.stdout:2095: copy_from oid 14 from oid 5 current snap is 208 2024-09-14T03:44:47.760 INFO:tasks.rados.rados.0.smithi110.stdout:2096: setattr oid 37 current snap is 208 2024-09-14T03:44:47.761 INFO:tasks.rados.rados.0.smithi110.stdout:2097: delete oid 36 current snap is 208 2024-09-14T03:44:47.762 INFO:tasks.rados.rados.0.smithi110.stdout:2088: expect (ObjNum 643 snap 199 seq_num 643) 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1479 (ObjNum 592 snap 181 seq_num 592) dirty exists 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:2095: finishing copy_from racing read to smithi11075562-14 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:2095: finishing copy_from to smithi11075562-14 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1477 (ObjNum 542 snap 166 seq_num 542) dirty exists 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:2088: done (3 left) 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:2095: done (2 left) 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:2096: done (1 left) 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:2097: done (0 left) 2024-09-14T03:44:47.834 INFO:tasks.rados.rados.0.smithi110.stdout:2098: snap_create 2024-09-14T03:44:47.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:44:47 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:44:47.664+0000 7f940e88b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:44:48.738 INFO:tasks.rados.rados.0.smithi110.stdout:2098: done (0 left) 2024-09-14T03:44:48.738 INFO:tasks.rados.rados.0.smithi110.stdout:2099: snap_remove snap 208 2024-09-14T03:44:48.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:48.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:44:48.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[88262]: pgmap v359: 121 pgs: 1 active+clean+snaptrim, 18 activating, 2 peering, 23 active+undersized, 5 active+undersized+degraded, 72 active+clean; 229 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 6 op/s; 36/411 objects degraded (8.759%) 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T03:44:48.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:48 smithi110 ceph-mon[86423]: pgmap v359: 121 pgs: 1 active+clean+snaptrim, 18 activating, 2 peering, 23 active+undersized, 5 active+undersized+degraded, 72 active+clean; 229 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 6 op/s; 36/411 objects degraded (8.759%) 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:44:48.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:44:48.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:44:48.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:44:48.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T03:44:48.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:48 smithi195 ceph-mon[82388]: pgmap v359: 121 pgs: 1 active+clean+snaptrim, 18 activating, 2 peering, 23 active+undersized, 5 active+undersized+degraded, 72 active+clean; 229 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 6 op/s; 36/411 objects degraded (8.759%) 2024-09-14T03:44:49.740 INFO:tasks.rados.rados.0.smithi110.stdout:2099: done (0 left) 2024-09-14T03:44:49.740 INFO:tasks.rados.rados.0.smithi110.stdout:2100: snap_remove snap 195 2024-09-14T03:44:49.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:49 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 59/402 objects degraded (14.677%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:49.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:49 smithi110 ceph-mon[86423]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T03:44:49.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:49 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 59/402 objects degraded (14.677%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:49.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:49 smithi110 ceph-mon[88262]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T03:44:49.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:49 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 59/402 objects degraded (14.677%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:44:49.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:49 smithi195 ceph-mon[82388]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T03:44:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:44:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:44:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:44:50.747 INFO:tasks.rados.rados.0.smithi110.stdout:2100: done (0 left) 2024-09-14T03:44:50.747 INFO:tasks.rados.rados.0.smithi110.stdout:2101: snap_remove snap 205 2024-09-14T03:44:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:50 smithi110 ceph-mon[88262]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T03:44:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:50 smithi110 ceph-mon[88262]: pgmap v362: 121 pgs: 1 active+clean+snaptrim, 18 activating, 2 peering, 17 active+undersized, 2 active+undersized+degraded, 81 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 784 KiB/s wr, 3 op/s; 21/408 objects degraded (5.147%); 8.3 MiB/s, 1 objects/s recovering 2024-09-14T03:44:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:50 smithi110 ceph-mon[86423]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T03:44:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:50 smithi110 ceph-mon[86423]: pgmap v362: 121 pgs: 1 active+clean+snaptrim, 18 activating, 2 peering, 17 active+undersized, 2 active+undersized+degraded, 81 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 784 KiB/s wr, 3 op/s; 21/408 objects degraded (5.147%); 8.3 MiB/s, 1 objects/s recovering 2024-09-14T03:44:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:50 smithi195 ceph-mon[82388]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T03:44:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:50 smithi195 ceph-mon[82388]: pgmap v362: 121 pgs: 1 active+clean+snaptrim, 18 activating, 2 peering, 17 active+undersized, 2 active+undersized+degraded, 81 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 784 KiB/s wr, 3 op/s; 21/408 objects degraded (5.147%); 8.3 MiB/s, 1 objects/s recovering 2024-09-14T03:44:51.749 INFO:tasks.rados.rados.0.smithi110.stdout:2101: done (0 left) 2024-09-14T03:44:51.749 INFO:tasks.rados.rados.0.smithi110.stdout:2102: delete oid 26 current snap is 209 2024-09-14T03:44:51.758 INFO:tasks.rados.rados.0.smithi110.stdout:2102: done (0 left) 2024-09-14T03:44:51.758 INFO:tasks.rados.rados.0.smithi110.stdout:2103: snap_remove snap 202 2024-09-14T03:44:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:51 smithi110 ceph-mon[86423]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T03:44:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:51 smithi110 ceph-mon[88262]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T03:44:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:44:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:44:51] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.51.0" 2024-09-14T03:44:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:51 smithi195 ceph-mon[82388]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T03:44:52.755 INFO:tasks.rados.rados.0.smithi110.stdout:2103: done (0 left) 2024-09-14T03:44:52.755 INFO:tasks.rados.rados.0.smithi110.stdout:2104: read oid 49 snap -1 2024-09-14T03:44:52.755 INFO:tasks.rados.rados.0.smithi110.stdout:2104: expect (ObjNum 660 snap 204 seq_num 660) 2024-09-14T03:44:52.756 INFO:tasks.rados.rados.0.smithi110.stdout:2105: copy_from oid 22 from oid 39 current snap is 209 2024-09-14T03:44:52.756 INFO:tasks.rados.rados.0.smithi110.stdout:2106: read oid 50 snap 206 2024-09-14T03:44:52.756 INFO:tasks.rados.rados.0.smithi110.stdout:2106: expect deleted 2024-09-14T03:44:52.756 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 209 2024-09-14T03:44:52.756 INFO:tasks.rados.rados.0.smithi110.stdout:2107: seq_num 674 ranges {1409024=491520,1900544=417792,2318336=483328,2801664=303104} 2024-09-14T03:44:52.767 INFO:tasks.rados.rados.0.smithi110.stdout:2107: writing smithi11075562-47 from 1409024 to 1900544 tid 1 2024-09-14T03:44:52.770 INFO:tasks.rados.rados.0.smithi110.stdout:2107: writing smithi11075562-47 from 1900544 to 2318336 tid 2 2024-09-14T03:44:52.773 INFO:tasks.rados.rados.0.smithi110.stdout:2107: writing smithi11075562-47 from 2318336 to 2801664 tid 3 2024-09-14T03:44:52.776 INFO:tasks.rados.rados.0.smithi110.stdout:2107: writing smithi11075562-47 from 2801664 to 3104768 tid 4 2024-09-14T03:44:52.776 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 209 2024-09-14T03:44:52.776 INFO:tasks.rados.rados.0.smithi110.stdout:2108: seq_num 675 ranges {2932736=507904,3440640=704512,4145152=499712} 2024-09-14T03:44:52.794 INFO:tasks.rados.rados.0.smithi110.stdout:2108: writing smithi11075562-33 from 2932736 to 3440640 tid 1 2024-09-14T03:44:52.798 INFO:tasks.rados.rados.0.smithi110.stdout:2108: writing smithi11075562-33 from 3440640 to 4145152 tid 2 2024-09-14T03:44:52.801 INFO:tasks.rados.rados.0.smithi110.stdout:2108: writing smithi11075562-33 from 4145152 to 4644864 tid 3 2024-09-14T03:44:52.801 INFO:tasks.rados.rados.0.smithi110.stdout:2109: rollback oid 1 current snap is 209 2024-09-14T03:44:52.801 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 1 to 207 2024-09-14T03:44:52.801 INFO:tasks.rados.rados.0.smithi110.stdout:2105: finishing copy_from to smithi11075562-22 2024-09-14T03:44:52.801 INFO:tasks.rados.rados.0.smithi110.stdout:2105: got expected ENOENT (src dne) 2024-09-14T03:44:52.801 INFO:tasks.rados.rados.0.smithi110.stdout:2105: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:44:52.802 INFO:tasks.rados.rados.0.smithi110.stdout:2104: expect (ObjNum 660 snap 204 seq_num 660) 2024-09-14T03:44:52.876 INFO:tasks.rados.rados.0.smithi110.stdout:2107: finishing write tid 1 to smithi11075562-47 2024-09-14T03:44:52.876 INFO:tasks.rados.rados.0.smithi110.stdout:2107: finishing write tid 2 to smithi11075562-47 2024-09-14T03:44:52.876 INFO:tasks.rados.rados.0.smithi110.stdout:2107: finishing write tid 3 to smithi11075562-47 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2107: finishing write tid 4 to smithi11075562-47 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2107: finishing write tid 5 to smithi11075562-47 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2107: finishing write tid 6 to smithi11075562-47 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1531 (ObjNum 674 snap 209 seq_num 674) dirty exists 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2107: left oid 47 (ObjNum 674 snap 209 seq_num 674) 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2108: finishing write tid 1 to smithi11075562-33 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2104: done (5 left) 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2105: done (4 left) 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2106: done (3 left) 2024-09-14T03:44:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2107: done (2 left) 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:2110: setattr oid 27 current snap is 209 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:2108: finishing write tid 2 to smithi11075562-33 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:2108: finishing write tid 3 to smithi11075562-33 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:2108: finishing write tid 4 to smithi11075562-33 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:2109: finishing rollback tid 0 to smithi11075562-1 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:2109: finishing rollback tid 1 to smithi11075562-1 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:2109: finishing rollback tid 2 to smithi11075562-1 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1276 (ObjNum 572 snap 174 seq_num 572) dirty exists 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:2108: finishing write tid 5 to smithi11075562-33 2024-09-14T03:44:52.878 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1451 (ObjNum 675 snap 209 seq_num 675) dirty exists 2024-09-14T03:44:52.879 INFO:tasks.rados.rados.0.smithi110.stdout:2108: left oid 33 (ObjNum 675 snap 209 seq_num 675) 2024-09-14T03:44:52.879 INFO:tasks.rados.rados.0.smithi110.stdout:2108: done (2 left) 2024-09-14T03:44:52.879 INFO:tasks.rados.rados.0.smithi110.stdout:2109: done (1 left) 2024-09-14T03:44:52.879 INFO:tasks.rados.rados.0.smithi110.stdout:2111: read oid 12 snap -1 2024-09-14T03:44:52.879 INFO:tasks.rados.rados.0.smithi110.stdout:2111: expect deleted 2024-09-14T03:44:52.880 INFO:tasks.rados.rados.0.smithi110.stdout:2112: snap_remove snap 193 2024-09-14T03:44:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:52 smithi110 ceph-mon[86423]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T03:44:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:52 smithi110 ceph-mon[86423]: pgmap v365: 121 pgs: 18 activating, 103 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 8 op/s; 3/408 objects degraded (0.735%); 13 MiB/s, 3 objects/s recovering 2024-09-14T03:44:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:52 smithi110 ceph-mon[88262]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T03:44:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:52 smithi110 ceph-mon[88262]: pgmap v365: 121 pgs: 18 activating, 103 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 8 op/s; 3/408 objects degraded (0.735%); 13 MiB/s, 3 objects/s recovering 2024-09-14T03:44:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:52 smithi195 ceph-mon[82388]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T03:44:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:52 smithi195 ceph-mon[82388]: pgmap v365: 121 pgs: 18 activating, 103 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 8 op/s; 3/408 objects degraded (0.735%); 13 MiB/s, 3 objects/s recovering 2024-09-14T03:44:53.209 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:44:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:44:52.767+0000 7f2bc2224640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 119 after trimming was started 2024-09-14T03:44:53.761 INFO:tasks.rados.rados.0.smithi110.stdout:2112: done (2 left) 2024-09-14T03:44:53.761 INFO:tasks.rados.rados.0.smithi110.stdout:2113: delete oid 26 current snap is 209 2024-09-14T03:44:53.761 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1201 (ObjNum 623 snap 190 seq_num 623) dirty exists 2024-09-14T03:44:53.778 INFO:tasks.rados.rados.0.smithi110.stdout:2110: done (2 left) 2024-09-14T03:44:53.778 INFO:tasks.rados.rados.0.smithi110.stdout:2111: done (1 left) 2024-09-14T03:44:53.778 INFO:tasks.rados.rados.0.smithi110.stdout:2113: done (0 left) 2024-09-14T03:44:53.778 INFO:tasks.rados.rados.0.smithi110.stdout:2114: snap_create 2024-09-14T03:44:54.082 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:53 smithi195 ceph-mon[82388]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/408 objects degraded (5.147%), 2 pgs degraded) 2024-09-14T03:44:54.082 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:53 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:44:54.082 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:53 smithi195 ceph-mon[82388]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T03:44:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:53 smithi110 ceph-mon[88262]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/408 objects degraded (5.147%), 2 pgs degraded) 2024-09-14T03:44:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:53 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:44:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:53 smithi110 ceph-mon[88262]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T03:44:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:53 smithi110 ceph-mon[86423]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/408 objects degraded (5.147%), 2 pgs degraded) 2024-09-14T03:44:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:53 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:44:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:53 smithi110 ceph-mon[86423]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T03:44:54.767 INFO:tasks.rados.rados.0.smithi110.stdout:2114: done (0 left) 2024-09-14T03:44:54.767 INFO:tasks.rados.rados.0.smithi110.stdout:2115: read oid 15 snap 207 2024-09-14T03:44:54.767 INFO:tasks.rados.rados.0.smithi110.stdout:2115: expect deleted 2024-09-14T03:44:54.768 INFO:tasks.rados.rados.0.smithi110.stdout:2116: setattr oid 3 current snap is 210 2024-09-14T03:44:54.768 INFO:tasks.rados.rados.0.smithi110.stdout:2115: done (1 left) 2024-09-14T03:44:54.768 INFO:tasks.rados.rados.0.smithi110.stdout:2117: snap_remove snap 209 2024-09-14T03:44:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:54 smithi110 ceph-mon[86423]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T03:44:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:54 smithi110 ceph-mon[86423]: pgmap v368: 121 pgs: 121 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 904 KiB/s wr, 7 op/s; 6.9 MiB/s, 2 objects/s recovering 2024-09-14T03:44:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:54 smithi110 ceph-mon[88262]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T03:44:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:54 smithi110 ceph-mon[88262]: pgmap v368: 121 pgs: 121 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 904 KiB/s wr, 7 op/s; 6.9 MiB/s, 2 objects/s recovering 2024-09-14T03:44:55.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:54 smithi195 ceph-mon[82388]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T03:44:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:54 smithi195 ceph-mon[82388]: pgmap v368: 121 pgs: 121 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 904 KiB/s wr, 7 op/s; 6.9 MiB/s, 2 objects/s recovering 2024-09-14T03:44:55.769 INFO:tasks.rados.rados.0.smithi110.stdout:2117: done (1 left) 2024-09-14T03:44:55.769 INFO:tasks.rados.rados.0.smithi110.stdout:2118: read oid 42 snap 207 2024-09-14T03:44:55.769 INFO:tasks.rados.rados.0.smithi110.stdout:2118: expect (ObjNum 643 snap 199 seq_num 643) 2024-09-14T03:44:55.769 INFO:tasks.rados.rados.0.smithi110.stdout:2119: read oid 28 snap 196 2024-09-14T03:44:55.770 INFO:tasks.rados.rados.0.smithi110.stdout:2119: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:44:55.770 INFO:tasks.rados.rados.0.smithi110.stdout:2120: snap_create 2024-09-14T03:44:55.770 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 775 (ObjNum 638 snap 197 seq_num 638) dirty exists 2024-09-14T03:44:55.779 INFO:tasks.rados.rados.0.smithi110.stdout:2118: expect (ObjNum 643 snap 199 seq_num 643) 2024-09-14T03:44:55.845 INFO:tasks.rados.rados.0.smithi110.stdout:2119: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:44:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:55 smithi110 ceph-mon[86423]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T03:44:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:55 smithi110 ceph-mon[88262]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T03:44:56.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:55 smithi195 ceph-mon[82388]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T03:44:56.779 INFO:tasks.rados.rados.0.smithi110.stdout:2116: done (3 left) 2024-09-14T03:44:56.779 INFO:tasks.rados.rados.0.smithi110.stdout:2118: done (2 left) 2024-09-14T03:44:56.779 INFO:tasks.rados.rados.0.smithi110.stdout:2119: done (1 left) 2024-09-14T03:44:56.779 INFO:tasks.rados.rados.0.smithi110.stdout:2120: done (0 left) 2024-09-14T03:44:56.780 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 211 2024-09-14T03:44:56.780 INFO:tasks.rados.rados.0.smithi110.stdout:2121: seq_num 677 ranges {12435456=540672,12976128=516096,13492224=655360,14147584=32768} 2024-09-14T03:44:56.847 INFO:tasks.rados.rados.0.smithi110.stdout:2121: writing smithi11075562-22 from 12435456 to 12976128 tid 1 2024-09-14T03:44:56.850 INFO:tasks.rados.rados.0.smithi110.stdout:2121: writing smithi11075562-22 from 12976128 to 13492224 tid 2 2024-09-14T03:44:56.854 INFO:tasks.rados.rados.0.smithi110.stdout:2121: writing smithi11075562-22 from 13492224 to 14147584 tid 3 2024-09-14T03:44:56.854 INFO:tasks.rados.rados.0.smithi110.stdout:2121: writing smithi11075562-22 from 14147584 to 14180352 tid 4 2024-09-14T03:44:56.854 INFO:tasks.rados.rados.0.smithi110.stdout:2122: copy_from oid 50 from oid 15 current snap is 211 2024-09-14T03:44:56.854 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 211 2024-09-14T03:44:56.854 INFO:tasks.rados.rados.0.smithi110.stdout:2123: seq_num 678 ranges {2056192=565248,2621440=802816,3424256=401408} 2024-09-14T03:44:56.869 INFO:tasks.rados.rados.0.smithi110.stdout:2123: writing smithi11075562-8 from 2056192 to 2621440 tid 1 2024-09-14T03:44:56.873 INFO:tasks.rados.rados.0.smithi110.stdout:2123: writing smithi11075562-8 from 2621440 to 3424256 tid 2 2024-09-14T03:44:56.876 INFO:tasks.rados.rados.0.smithi110.stdout:2123: writing smithi11075562-8 from 3424256 to 3825664 tid 3 2024-09-14T03:44:56.876 INFO:tasks.rados.rados.0.smithi110.stdout:2124: copy_from oid 31 from oid 27 current snap is 211 2024-09-14T03:44:56.876 INFO:tasks.rados.rados.0.smithi110.stdout:2121: finishing write tid 1 to smithi11075562-22 2024-09-14T03:44:56.876 INFO:tasks.rados.rados.0.smithi110.stdout:2121: finishing write tid 2 to smithi11075562-22 2024-09-14T03:44:56.876 INFO:tasks.rados.rados.0.smithi110.stdout:2121: finishing write tid 3 to smithi11075562-22 2024-09-14T03:44:56.876 INFO:tasks.rados.rados.0.smithi110.stdout:2121: finishing write tid 4 to smithi11075562-22 2024-09-14T03:44:56.876 INFO:tasks.rados.rados.0.smithi110.stdout:2122: finishing copy_from to smithi11075562-50 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:2122: got expected ENOENT (src dne) 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:2122: finishing copy_from racing read to smithi11075562-50 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:2121: finishing write tid 5 to smithi11075562-22 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:2121: finishing write tid 6 to smithi11075562-22 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1283 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:2121: left oid 22 (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:2123: finishing write tid 1 to smithi11075562-8 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:2121: done (3 left) 2024-09-14T03:44:56.877 INFO:tasks.rados.rados.0.smithi110.stdout:2122: done (2 left) 2024-09-14T03:44:56.878 INFO:tasks.rados.rados.0.smithi110.stdout:2125: snap_create 2024-09-14T03:44:56.878 INFO:tasks.rados.rados.0.smithi110.stdout:2123: finishing write tid 2 to smithi11075562-8 2024-09-14T03:44:56.880 INFO:tasks.rados.rados.0.smithi110.stdout:2123: finishing write tid 3 to smithi11075562-8 2024-09-14T03:44:56.880 INFO:tasks.rados.rados.0.smithi110.stdout:2123: finishing write tid 4 to smithi11075562-8 2024-09-14T03:44:56.881 INFO:tasks.rados.rados.0.smithi110.stdout:2123: finishing write tid 5 to smithi11075562-8 2024-09-14T03:44:56.882 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1288 (ObjNum 678 snap 211 seq_num 678) dirty exists 2024-09-14T03:44:56.882 INFO:tasks.rados.rados.0.smithi110.stdout:2123: left oid 8 (ObjNum 678 snap 211 seq_num 678) 2024-09-14T03:44:56.901 INFO:tasks.rados.rados.0.smithi110.stdout:2124: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:44:56.909 INFO:tasks.rados.rados.0.smithi110.stdout:2124: finishing copy_from to smithi11075562-31 2024-09-14T03:44:56.909 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1201 (ObjNum 623 snap 190 seq_num 623) dirty exists 2024-09-14T03:44:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:56 smithi110 ceph-mon[86423]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T03:44:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:56 smithi110 ceph-mon[86423]: pgmap v371: 121 pgs: 121 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 7 op/s; 1.8 MiB/s, 0 objects/s recovering 2024-09-14T03:44:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:56 smithi110 ceph-mon[88262]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T03:44:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:56 smithi110 ceph-mon[88262]: pgmap v371: 121 pgs: 121 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 7 op/s; 1.8 MiB/s, 0 objects/s recovering 2024-09-14T03:44:57.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:56 smithi195 ceph-mon[82388]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T03:44:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:56 smithi195 ceph-mon[82388]: pgmap v371: 121 pgs: 121 active+clean; 218 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 7 op/s; 1.8 MiB/s, 0 objects/s recovering 2024-09-14T03:44:57.784 INFO:tasks.rados.rados.0.smithi110.stdout:2123: done (2 left) 2024-09-14T03:44:57.784 INFO:tasks.rados.rados.0.smithi110.stdout:2124: done (1 left) 2024-09-14T03:44:57.784 INFO:tasks.rados.rados.0.smithi110.stdout:2125: done (0 left) 2024-09-14T03:44:57.784 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 212 2024-09-14T03:44:57.784 INFO:tasks.rados.rados.0.smithi110.stdout:2126: seq_num 679 ranges {4710400=581632,5292032=688128,5980160=516096} 2024-09-14T03:44:57.811 INFO:tasks.rados.rados.0.smithi110.stdout:2126: writing smithi11075562-43 from 4710400 to 5292032 tid 1 2024-09-14T03:44:57.815 INFO:tasks.rados.rados.0.smithi110.stdout:2126: writing smithi11075562-43 from 5292032 to 5980160 tid 2 2024-09-14T03:44:57.818 INFO:tasks.rados.rados.0.smithi110.stdout:2126: writing smithi11075562-43 from 5980160 to 6496256 tid 3 2024-09-14T03:44:57.818 INFO:tasks.rados.rados.0.smithi110.stdout:2127: delete oid 18 current snap is 212 2024-09-14T03:44:57.819 INFO:tasks.rados.rados.0.smithi110.stdout:2126: finishing write tid 1 to smithi11075562-43 2024-09-14T03:44:57.821 INFO:tasks.rados.rados.0.smithi110.stdout:2126: finishing write tid 2 to smithi11075562-43 2024-09-14T03:44:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:2127: done (1 left) 2024-09-14T03:44:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:2128: read oid 13 snap -1 2024-09-14T03:44:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:2128: expect (ObjNum 550 snap 167 seq_num 550) 2024-09-14T03:44:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 212 2024-09-14T03:44:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:2126: finishing write tid 3 to smithi11075562-43 2024-09-14T03:44:57.822 INFO:tasks.rados.rados.0.smithi110.stdout:2129: seq_num 681 ranges {3825664=614400,4440064=458752,4898816=638976,5537792=106496} 2024-09-14T03:44:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:2129: writing smithi11075562-8 from 3825664 to 4440064 tid 1 2024-09-14T03:44:57.849 INFO:tasks.rados.rados.0.smithi110.stdout:2129: writing smithi11075562-8 from 4440064 to 4898816 tid 2 2024-09-14T03:44:57.853 INFO:tasks.rados.rados.0.smithi110.stdout:2129: writing smithi11075562-8 from 4898816 to 5537792 tid 3 2024-09-14T03:44:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:2129: writing smithi11075562-8 from 5537792 to 5644288 tid 4 2024-09-14T03:44:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:2130: snap_create 2024-09-14T03:44:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:2126: finishing write tid 4 to smithi11075562-43 2024-09-14T03:44:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:2126: finishing write tid 5 to smithi11075562-43 2024-09-14T03:44:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1481 (ObjNum 679 snap 212 seq_num 679) dirty exists 2024-09-14T03:44:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:2126: left oid 43 (ObjNum 679 snap 212 seq_num 679) 2024-09-14T03:44:57.855 INFO:tasks.rados.rados.0.smithi110.stdout:2128: expect (ObjNum 550 snap 167 seq_num 550) 2024-09-14T03:44:58.135 INFO:tasks.rados.rados.0.smithi110.stdout:2129: finishing write tid 1 to smithi11075562-8 2024-09-14T03:44:58.135 INFO:tasks.rados.rados.0.smithi110.stdout:2129: finishing write tid 2 to smithi11075562-8 2024-09-14T03:44:58.135 INFO:tasks.rados.rados.0.smithi110.stdout:2129: finishing write tid 3 to smithi11075562-8 2024-09-14T03:44:58.135 INFO:tasks.rados.rados.0.smithi110.stdout:2129: finishing write tid 4 to smithi11075562-8 2024-09-14T03:44:58.135 INFO:tasks.rados.rados.0.smithi110.stdout:2129: finishing write tid 5 to smithi11075562-8 2024-09-14T03:44:58.135 INFO:tasks.rados.rados.0.smithi110.stdout:2129: finishing write tid 6 to smithi11075562-8 2024-09-14T03:44:58.135 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1294 (ObjNum 681 snap 212 seq_num 681) dirty exists 2024-09-14T03:44:58.135 INFO:tasks.rados.rados.0.smithi110.stdout:2129: left oid 8 (ObjNum 681 snap 212 seq_num 681) 2024-09-14T03:44:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:57 smithi110 ceph-mon[88262]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T03:44:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:57 smithi110 ceph-mon[86423]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T03:44:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:58.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:57 smithi195 ceph-mon[82388]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T03:44:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:44:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:44:58.549 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:44:58.724 INFO:tasks.rados.rados.0.smithi110.stdout:2126: done (3 left) 2024-09-14T03:44:58.724 INFO:tasks.rados.rados.0.smithi110.stdout:2128: done (2 left) 2024-09-14T03:44:58.724 INFO:tasks.rados.rados.0.smithi110.stdout:2129: done (1 left) 2024-09-14T03:44:58.724 INFO:tasks.rados.rados.0.smithi110.stdout:2130: done (0 left) 2024-09-14T03:44:58.725 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 213 2024-09-14T03:44:58.725 INFO:tasks.rados.rados.0.smithi110.stdout:2131: seq_num 682 ranges {0=630784,630784=745472,1376256=442368,1818624=16384} 2024-09-14T03:44:58.727 INFO:tasks.rados.rados.0.smithi110.stdout:2131: writing smithi11075562-18 from 0 to 630784 tid 1 2024-09-14T03:44:58.732 INFO:tasks.rados.rados.0.smithi110.stdout:2131: writing smithi11075562-18 from 630784 to 1376256 tid 2 2024-09-14T03:44:58.736 INFO:tasks.rados.rados.0.smithi110.stdout:2131: writing smithi11075562-18 from 1376256 to 1818624 tid 3 2024-09-14T03:44:58.736 INFO:tasks.rados.rados.0.smithi110.stdout:2131: writing smithi11075562-18 from 1818624 to 1835008 tid 4 2024-09-14T03:44:58.736 INFO:tasks.rados.rados.0.smithi110.stdout:2132: snap_create 2024-09-14T03:44:58.736 INFO:tasks.rados.rados.0.smithi110.stdout:2131: finishing write tid 1 to smithi11075562-18 2024-09-14T03:44:58.736 INFO:tasks.rados.rados.0.smithi110.stdout:2131: finishing write tid 2 to smithi11075562-18 2024-09-14T03:44:58.739 INFO:tasks.rados.rados.0.smithi110.stdout:2131: finishing write tid 3 to smithi11075562-18 2024-09-14T03:44:58.740 INFO:tasks.rados.rados.0.smithi110.stdout:2131: finishing write tid 4 to smithi11075562-18 2024-09-14T03:44:58.741 INFO:tasks.rados.rados.0.smithi110.stdout:2131: finishing write tid 5 to smithi11075562-18 2024-09-14T03:44:58.742 INFO:tasks.rados.rados.0.smithi110.stdout:2131: finishing write tid 6 to smithi11075562-18 2024-09-14T03:44:58.742 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1163 (ObjNum 682 snap 213 seq_num 682) dirty exists 2024-09-14T03:44:58.742 INFO:tasks.rados.rados.0.smithi110.stdout:2131: left oid 18 (ObjNum 682 snap 213 seq_num 682) 2024-09-14T03:44:58.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:58 smithi110 ceph-mon[88262]: pgmap v373: 121 pgs: 121 active+clean; 203 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 829 KiB/s wr, 10 op/s 2024-09-14T03:44:58.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:58 smithi110 ceph-mon[88262]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T03:44:58.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:58 smithi110 ceph-mon[88262]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T03:44:58.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:58 smithi110 ceph-mon[86423]: pgmap v373: 121 pgs: 121 active+clean; 203 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 829 KiB/s wr, 10 op/s 2024-09-14T03:44:58.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:58 smithi110 ceph-mon[86423]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T03:44:58.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:58 smithi110 ceph-mon[86423]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T03:44:59.110 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:44:59.110 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (5m) 83s ago 9m 19.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (5m) 15s ago 9m 84.7M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (11m) 83s ago 11m 20.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (11m) 15s ago 11m 17.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (4m) 15s ago 14m 454M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (5m) 83s ago 16m 525M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (4m) 83s ago 16m 55.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (3m) 15s ago 14m 46.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (4m) 83s ago 14m 49.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (5m) 83s ago 9m 9.92M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (5m) 15s ago 9m 10.7M - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (3m) 83s ago 14m 218M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (2m) 83s ago 13m 117M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:44:59.111 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (2m) 83s ago 13m 101M 2480M 19.3.0-4907-ga2782315 8801d2529e43 0cdc58c38711 2024-09-14T03:44:59.112 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (87s) 83s ago 13m 14.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2283ebcb5f10 2024-09-14T03:44:59.112 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (54s) 15s ago 12m 146M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6921b0a30047 2024-09-14T03:44:59.112 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (20s) 15s ago 12m 14.5M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8fe7f08d5882 2024-09-14T03:44:59.112 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (11m) 15s ago 11m 222M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 f979f8dd4ea0 2024-09-14T03:44:59.112 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (11m) 15s ago 11m 180M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:44:59.112 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (4m) 15s ago 10m 52.7M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:44:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:58 smithi195 ceph-mon[82388]: pgmap v373: 121 pgs: 121 active+clean; 203 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 829 KiB/s wr, 10 op/s 2024-09-14T03:44:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:58 smithi195 ceph-mon[82388]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T03:44:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:58 smithi195 ceph-mon[82388]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T03:44:59.442 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:44:59.442 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:44:59.442 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:44:59.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2, 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 6 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 4, 2024-09-14T03:44:59.443 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 11 2024-09-14T03:44:59.444 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:44:59.444 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:44:59.724 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "11/20 daemons upgraded", 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:44:59.725 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:44:59.730 INFO:tasks.rados.rados.0.smithi110.stdout:2131: done (1 left) 2024-09-14T03:44:59.730 INFO:tasks.rados.rados.0.smithi110.stdout:2132: done (0 left) 2024-09-14T03:44:59.730 INFO:tasks.rados.rados.0.smithi110.stdout:2133: setattr oid 44 current snap is 214 2024-09-14T03:44:59.731 INFO:tasks.rados.rados.0.smithi110.stdout:2134: rollback oid 20 current snap is 214 2024-09-14T03:44:59.731 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 207 2024-09-14T03:44:59.732 INFO:tasks.rados.rados.0.smithi110.stdout:2135: snap_remove snap 211 2024-09-14T03:45:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:59 smithi110 ceph-mon[88262]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:59 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1419787750' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:44:59 smithi110 ceph-mon[88262]: osdmap e499: 8 total, 8 up, 8 in 2024-09-14T03:45:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:59 smithi110 ceph-mon[86423]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:59 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1419787750' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:00.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:44:59 smithi110 ceph-mon[86423]: osdmap e499: 8 total, 8 up, 8 in 2024-09-14T03:45:00.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:59 smithi195 ceph-mon[82388]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:59 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1419787750' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:44:59 smithi195 ceph-mon[82388]: osdmap e499: 8 total, 8 up, 8 in 2024-09-14T03:45:00.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:45:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:45:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:45:00.737 INFO:tasks.rados.rados.0.smithi110.stdout:2135: done (2 left) 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:2136: copy_from oid 39 from oid 18 current snap is 214 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 401 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:2134: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:2134: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:2134: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1454 (ObjNum 663 snap 204 seq_num 663) dirty exists 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:2133: done (2 left) 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:2134: done (1 left) 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 214 2024-09-14T03:45:00.738 INFO:tasks.rados.rados.0.smithi110.stdout:2137: seq_num 683 ranges {2752512=647168,3399680=630784,4030464=573440} 2024-09-14T03:45:00.757 INFO:tasks.rados.rados.0.smithi110.stdout:2137: writing smithi11075562-23 from 2752512 to 3399680 tid 1 2024-09-14T03:45:00.761 INFO:tasks.rados.rados.0.smithi110.stdout:2137: writing smithi11075562-23 from 3399680 to 4030464 tid 2 2024-09-14T03:45:00.765 INFO:tasks.rados.rados.0.smithi110.stdout:2137: writing smithi11075562-23 from 4030464 to 4603904 tid 3 2024-09-14T03:45:00.765 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 214 2024-09-14T03:45:00.765 INFO:tasks.rados.rados.0.smithi110.stdout:2136: finishing copy_from racing read to smithi11075562-39 2024-09-14T03:45:00.765 INFO:tasks.rados.rados.0.smithi110.stdout:2136: finishing copy_from to smithi11075562-39 2024-09-14T03:45:00.765 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1163 (ObjNum 682 snap 213 seq_num 682) dirty exists 2024-09-14T03:45:00.765 INFO:tasks.rados.rados.0.smithi110.stdout:2137: finishing write tid 1 to smithi11075562-23 2024-09-14T03:45:00.765 INFO:tasks.rados.rados.0.smithi110.stdout:2137: finishing write tid 2 to smithi11075562-23 2024-09-14T03:45:00.765 INFO:tasks.rados.rados.0.smithi110.stdout:2138: seq_num 684 ranges {1343488=663552,2007040=507904,2514944=409600,2924544=286720} 2024-09-14T03:45:00.775 INFO:tasks.rados.rados.0.smithi110.stdout:2138: writing smithi11075562-30 from 1343488 to 2007040 tid 1 2024-09-14T03:45:00.778 INFO:tasks.rados.rados.0.smithi110.stdout:2138: writing smithi11075562-30 from 2007040 to 2514944 tid 2 2024-09-14T03:45:00.781 INFO:tasks.rados.rados.0.smithi110.stdout:2138: writing smithi11075562-30 from 2514944 to 2924544 tid 3 2024-09-14T03:45:00.783 INFO:tasks.rados.rados.0.smithi110.stdout:2138: writing smithi11075562-30 from 2924544 to 3211264 tid 4 2024-09-14T03:45:00.783 INFO:tasks.rados.rados.0.smithi110.stdout:2136: done (2 left) 2024-09-14T03:45:00.784 INFO:tasks.rados.rados.0.smithi110.stdout:2139: read oid 2 snap -1 2024-09-14T03:45:00.784 INFO:tasks.rados.rados.0.smithi110.stdout:2139: expect deleted 2024-09-14T03:45:00.784 INFO:tasks.rados.rados.0.smithi110.stdout:2140: read oid 46 snap -1 2024-09-14T03:45:00.784 INFO:tasks.rados.rados.0.smithi110.stdout:2140: expect (ObjNum 628 snap 195 seq_num 628) 2024-09-14T03:45:00.784 INFO:tasks.rados.rados.0.smithi110.stdout:2141: snap_remove snap 199 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[88262]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[88262]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[88262]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[88262]: pgmap v377: 121 pgs: 121 active+clean; 215 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.3 MiB/s wr, 14 op/s 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[88262]: osdmap e500: 8 total, 8 up, 8 in 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[86423]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[86423]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[86423]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[86423]: pgmap v377: 121 pgs: 121 active+clean; 215 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.3 MiB/s wr, 14 op/s 2024-09-14T03:45:01.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:00 smithi110 ceph-mon[86423]: osdmap e500: 8 total, 8 up, 8 in 2024-09-14T03:45:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:00 smithi195 ceph-mon[82388]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:00 smithi195 ceph-mon[82388]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:00 smithi195 ceph-mon[82388]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:00 smithi195 ceph-mon[82388]: pgmap v377: 121 pgs: 121 active+clean; 215 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.3 MiB/s wr, 14 op/s 2024-09-14T03:45:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:00 smithi195 ceph-mon[82388]: osdmap e500: 8 total, 8 up, 8 in 2024-09-14T03:45:01.739 INFO:tasks.rados.rados.0.smithi110.stdout:2141: done (4 left) 2024-09-14T03:45:01.739 INFO:tasks.rados.rados.0.smithi110.stdout:2142: rollback oid 33 current snap is 214 2024-09-14T03:45:01.739 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 33 to 207 2024-09-14T03:45:01.739 INFO:tasks.rados.rados.0.smithi110.stdout:2137: finishing write tid 3 to smithi11075562-23 2024-09-14T03:45:01.739 INFO:tasks.rados.rados.0.smithi110.stdout:2137: finishing write tid 4 to smithi11075562-23 2024-09-14T03:45:01.739 INFO:tasks.rados.rados.0.smithi110.stdout:2137: finishing write tid 5 to smithi11075562-23 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 942 (ObjNum 683 snap 214 seq_num 683) dirty exists 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2137: left oid 23 (ObjNum 683 snap 214 seq_num 683) 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2138: finishing write tid 1 to smithi11075562-30 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2138: finishing write tid 2 to smithi11075562-30 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2138: finishing write tid 3 to smithi11075562-30 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2138: finishing write tid 4 to smithi11075562-30 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2138: finishing write tid 5 to smithi11075562-30 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2138: finishing write tid 6 to smithi11075562-30 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1302 (ObjNum 684 snap 214 seq_num 684) dirty exists 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2138: left oid 30 (ObjNum 684 snap 214 seq_num 684) 2024-09-14T03:45:01.740 INFO:tasks.rados.rados.0.smithi110.stdout:2140: expect (ObjNum 628 snap 195 seq_num 628) 2024-09-14T03:45:01.775 INFO:tasks.rados.rados.0.smithi110.stdout:2142: finishing rollback tid 0 to smithi11075562-33 2024-09-14T03:45:01.775 INFO:tasks.rados.rados.0.smithi110.stdout:2142: finishing rollback tid 1 to smithi11075562-33 2024-09-14T03:45:01.775 INFO:tasks.rados.rados.0.smithi110.stdout:2137: done (4 left) 2024-09-14T03:45:01.775 INFO:tasks.rados.rados.0.smithi110.stdout:2138: done (3 left) 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2139: done (2 left) 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2140: done (1 left) 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2143: read oid 31 snap -1 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2143: expect (ObjNum 623 snap 190 seq_num 623) 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2144: copy_from oid 37 from oid 21 current snap is 214 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2142: finishing rollback tid 2 to smithi11075562-33 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1457 (ObjNum 669 snap 207 seq_num 669) dirty exists 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2142: done (2 left) 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2145: read oid 43 snap 213 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2145: expect (ObjNum 679 snap 212 seq_num 679) 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2146: read oid 11 snap -1 2024-09-14T03:45:01.776 INFO:tasks.rados.rados.0.smithi110.stdout:2146: expect (ObjNum 604 snap 186 seq_num 604) 2024-09-14T03:45:01.777 INFO:tasks.rados.rados.0.smithi110.stdout:2147: read oid 44 snap -1 2024-09-14T03:45:01.777 INFO:tasks.rados.rados.0.smithi110.stdout:2147: expect (ObjNum 553 snap 168 seq_num 553) 2024-09-14T03:45:01.777 INFO:tasks.rados.rados.0.smithi110.stdout:2148: snap_create 2024-09-14T03:45:01.780 INFO:tasks.rados.rados.0.smithi110.stdout:2146: expect (ObjNum 604 snap 186 seq_num 604) 2024-09-14T03:45:01.797 INFO:tasks.rados.rados.0.smithi110.stdout:2144: finishing copy_from to smithi11075562-37 2024-09-14T03:45:01.797 INFO:tasks.rados.rados.0.smithi110.stdout:2144: got expected ENOENT (src dne) 2024-09-14T03:45:01.797 INFO:tasks.rados.rados.0.smithi110.stdout:2144: finishing copy_from racing read to smithi11075562-37 2024-09-14T03:45:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:2145: expect (ObjNum 679 snap 212 seq_num 679) 2024-09-14T03:45:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:45:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:45:01] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.51.0" 2024-09-14T03:45:02.158 INFO:tasks.rados.rados.0.smithi110.stdout:2143: expect (ObjNum 623 snap 190 seq_num 623) 2024-09-14T03:45:02.734 INFO:tasks.rados.rados.0.smithi110.stdout:2147: expect (ObjNum 553 snap 168 seq_num 553) 2024-09-14T03:45:02.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:02 smithi110 ceph-mon[88262]: osdmap e501: 8 total, 8 up, 8 in 2024-09-14T03:45:02.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:02 smithi110 ceph-mon[88262]: pgmap v380: 121 pgs: 121 active+clean; 217 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.4 MiB/s wr, 13 op/s 2024-09-14T03:45:02.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:02 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:02.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:02 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:45:02.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:02 smithi110 ceph-mon[86423]: osdmap e501: 8 total, 8 up, 8 in 2024-09-14T03:45:02.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:02 smithi110 ceph-mon[86423]: pgmap v380: 121 pgs: 121 active+clean; 217 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.4 MiB/s wr, 13 op/s 2024-09-14T03:45:02.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:02 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:02.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:02 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:45:03.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:02 smithi195 ceph-mon[82388]: osdmap e501: 8 total, 8 up, 8 in 2024-09-14T03:45:03.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:02 smithi195 ceph-mon[82388]: pgmap v380: 121 pgs: 121 active+clean; 217 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.4 MiB/s wr, 13 op/s 2024-09-14T03:45:03.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:02 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:03.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:02 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:45:03.486 INFO:tasks.rados.rados.0.smithi110.stdout:2143: done (5 left) 2024-09-14T03:45:03.486 INFO:tasks.rados.rados.0.smithi110.stdout:2144: done (4 left) 2024-09-14T03:45:03.486 INFO:tasks.rados.rados.0.smithi110.stdout:2145: done (3 left) 2024-09-14T03:45:03.486 INFO:tasks.rados.rados.0.smithi110.stdout:2146: done (2 left) 2024-09-14T03:45:03.486 INFO:tasks.rados.rados.0.smithi110.stdout:2147: done (1 left) 2024-09-14T03:45:03.486 INFO:tasks.rados.rados.0.smithi110.stdout:2148: done (0 left) 2024-09-14T03:45:03.486 INFO:tasks.rados.rados.0.smithi110.stdout:2149: delete oid 44 current snap is 215 2024-09-14T03:45:03.489 INFO:tasks.rados.rados.0.smithi110.stdout:2149: done (0 left) 2024-09-14T03:45:03.489 INFO:tasks.rados.rados.0.smithi110.stdout:2150: copy_from oid 39 from oid 22 current snap is 215 2024-09-14T03:45:03.489 INFO:tasks.rados.rados.0.smithi110.stdout:2151: snap_remove snap 207 2024-09-14T03:45:03.726 INFO:tasks.rados.rados.0.smithi110.stdout:2151: done (1 left) 2024-09-14T03:45:03.727 INFO:tasks.rados.rados.0.smithi110.stdout:2152: read oid 14 snap -1 2024-09-14T03:45:03.727 INFO:tasks.rados.rados.0.smithi110.stdout:2152: expect (ObjNum 542 snap 166 seq_num 542) 2024-09-14T03:45:03.727 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 215 2024-09-14T03:45:03.727 INFO:tasks.rados.rados.0.smithi110.stdout:2150: finishing copy_from racing read to smithi11075562-39 2024-09-14T03:45:03.727 INFO:tasks.rados.rados.0.smithi110.stdout:2150: finishing copy_from to smithi11075562-39 2024-09-14T03:45:03.727 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1283 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:45:03.727 INFO:tasks.rados.rados.0.smithi110.stdout:2153: seq_num 686 ranges {0=696320,696320=729088,1425408=425984,1851392=49152} 2024-09-14T03:45:03.731 INFO:tasks.rados.rados.0.smithi110.stdout:2153: writing smithi11075562-26 from 0 to 696320 tid 1 2024-09-14T03:45:03.736 INFO:tasks.rados.rados.0.smithi110.stdout:2153: writing smithi11075562-26 from 696320 to 1425408 tid 2 2024-09-14T03:45:03.739 INFO:tasks.rados.rados.0.smithi110.stdout:2153: writing smithi11075562-26 from 1425408 to 1851392 tid 3 2024-09-14T03:45:03.741 INFO:tasks.rados.rados.0.smithi110.stdout:2153: writing smithi11075562-26 from 1851392 to 1900544 tid 4 2024-09-14T03:45:03.741 INFO:tasks.rados.rados.0.smithi110.stdout:2150: done (2 left) 2024-09-14T03:45:03.741 INFO:tasks.rados.rados.0.smithi110.stdout:2154: rollback oid 31 current snap is 215 2024-09-14T03:45:03.741 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 213 2024-09-14T03:45:03.741 INFO:tasks.rados.rados.0.smithi110.stdout:2155: rollback oid 12 current snap is 215 2024-09-14T03:45:03.741 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 213 2024-09-14T03:45:03.742 INFO:tasks.rados.rados.0.smithi110.stdout:2156: read oid 37 snap -1 2024-09-14T03:45:03.742 INFO:tasks.rados.rados.0.smithi110.stdout:2156: expect (ObjNum 592 snap 181 seq_num 592) 2024-09-14T03:45:03.742 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 215 2024-09-14T03:45:03.742 INFO:tasks.rados.rados.0.smithi110.stdout:2157: seq_num 687 ranges {3211264=712704,3923968=614400,4538368=589824} 2024-09-14T03:45:03.764 INFO:tasks.rados.rados.0.smithi110.stdout:2157: writing smithi11075562-30 from 3211264 to 3923968 tid 1 2024-09-14T03:45:03.767 INFO:tasks.rados.rados.0.smithi110.stdout:2157: writing smithi11075562-30 from 3923968 to 4538368 tid 2 2024-09-14T03:45:03.771 INFO:tasks.rados.rados.0.smithi110.stdout:2157: writing smithi11075562-30 from 4538368 to 5128192 tid 3 2024-09-14T03:45:03.771 INFO:tasks.rados.rados.0.smithi110.stdout:2158: snap_create 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2153: finishing write tid 1 to smithi11075562-26 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2153: finishing write tid 2 to smithi11075562-26 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2153: finishing write tid 3 to smithi11075562-26 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2153: finishing write tid 4 to smithi11075562-26 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2153: finishing write tid 5 to smithi11075562-26 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2154: finishing rollback tid 0 to smithi11075562-31 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2155: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 0 (ObjNum 33 snap 0 seq_num 1953066355) dirty dne 2024-09-14T03:45:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2152: expect (ObjNum 542 snap 166 seq_num 542) 2024-09-14T03:45:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:2154: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:45:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:2154: finishing rollback tid 2 to smithi11075562-31 2024-09-14T03:45:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1204 (ObjNum 623 snap 190 seq_num 623) dirty exists 2024-09-14T03:45:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:2153: finishing write tid 6 to smithi11075562-26 2024-09-14T03:45:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 785 (ObjNum 686 snap 215 seq_num 686) dirty exists 2024-09-14T03:45:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:2153: left oid 26 (ObjNum 686 snap 215 seq_num 686) 2024-09-14T03:45:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:2157: finishing write tid 1 to smithi11075562-30 2024-09-14T03:45:03.925 INFO:tasks.rados.rados.0.smithi110.stdout:2157: finishing write tid 2 to smithi11075562-30 2024-09-14T03:45:03.925 INFO:tasks.rados.rados.0.smithi110.stdout:2156: expect (ObjNum 592 snap 181 seq_num 592) 2024-09-14T03:45:04.094 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:03 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:45:04.094 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:03 smithi195 ceph-mon[82388]: Upgrade: osd.6 is safe to restart 2024-09-14T03:45:04.094 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:03 smithi195 ceph-mon[82388]: osdmap e502: 8 total, 8 up, 8 in 2024-09-14T03:45:04.094 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:03 smithi195 ceph-mon[82388]: osdmap e503: 8 total, 8 up, 8 in 2024-09-14T03:45:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:03 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:45:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:03 smithi110 ceph-mon[88262]: Upgrade: osd.6 is safe to restart 2024-09-14T03:45:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:03 smithi110 ceph-mon[88262]: osdmap e502: 8 total, 8 up, 8 in 2024-09-14T03:45:04.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:03 smithi110 ceph-mon[88262]: osdmap e503: 8 total, 8 up, 8 in 2024-09-14T03:45:04.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:03 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:45:04.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:03 smithi110 ceph-mon[86423]: Upgrade: osd.6 is safe to restart 2024-09-14T03:45:04.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:03 smithi110 ceph-mon[86423]: osdmap e502: 8 total, 8 up, 8 in 2024-09-14T03:45:04.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:03 smithi110 ceph-mon[86423]: osdmap e503: 8 total, 8 up, 8 in 2024-09-14T03:45:04.269 INFO:tasks.rados.rados.0.smithi110.stdout:2157: finishing write tid 3 to smithi11075562-30 2024-09-14T03:45:04.269 INFO:tasks.rados.rados.0.smithi110.stdout:2157: finishing write tid 4 to smithi11075562-30 2024-09-14T03:45:04.269 INFO:tasks.rados.rados.0.smithi110.stdout:2157: finishing write tid 5 to smithi11075562-30 2024-09-14T03:45:04.269 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1321 (ObjNum 687 snap 215 seq_num 687) dirty exists 2024-09-14T03:45:04.269 INFO:tasks.rados.rados.0.smithi110.stdout:2157: left oid 30 (ObjNum 687 snap 215 seq_num 687) 2024-09-14T03:45:04.749 INFO:tasks.rados.rados.0.smithi110.stdout:2152: done (6 left) 2024-09-14T03:45:04.749 INFO:tasks.rados.rados.0.smithi110.stdout:2153: done (5 left) 2024-09-14T03:45:04.749 INFO:tasks.rados.rados.0.smithi110.stdout:2154: done (4 left) 2024-09-14T03:45:04.749 INFO:tasks.rados.rados.0.smithi110.stdout:2155: done (3 left) 2024-09-14T03:45:04.749 INFO:tasks.rados.rados.0.smithi110.stdout:2156: done (2 left) 2024-09-14T03:45:04.750 INFO:tasks.rados.rados.0.smithi110.stdout:2157: done (1 left) 2024-09-14T03:45:04.750 INFO:tasks.rados.rados.0.smithi110.stdout:2158: done (0 left) 2024-09-14T03:45:04.750 INFO:tasks.rados.rados.0.smithi110.stdout:2159: read oid 50 snap -1 2024-09-14T03:45:04.750 INFO:tasks.rados.rados.0.smithi110.stdout:2159: expect deleted 2024-09-14T03:45:04.750 INFO:tasks.rados.rados.0.smithi110.stdout:2160: snap_remove snap 215 2024-09-14T03:45:05.094 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:04 smithi195 ceph-mon[82388]: pgmap v383: 121 pgs: 121 active+clean; 220 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-09-14T03:45:05.094 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:04 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:05.094 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:04 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:45:05.094 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:04 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:04 smithi110 ceph-mon[86423]: pgmap v383: 121 pgs: 121 active+clean; 220 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-09-14T03:45:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:04 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:04 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:45:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:04 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:04 smithi110 ceph-mon[88262]: pgmap v383: 121 pgs: 121 active+clean; 220 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-09-14T03:45:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:04 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:04 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:45:05.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:04 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:05.750 INFO:tasks.rados.rados.0.smithi110.stdout:2160: done (1 left) 2024-09-14T03:45:05.750 INFO:tasks.rados.rados.0.smithi110.stdout:2161: rollback oid 11 current snap is 216 2024-09-14T03:45:05.750 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 213 2024-09-14T03:45:05.750 INFO:tasks.rados.rados.0.smithi110.stdout:2159: done (1 left) 2024-09-14T03:45:05.750 INFO:tasks.rados.rados.0.smithi110.stdout:2162: read oid 34 snap -1 2024-09-14T03:45:05.750 INFO:tasks.rados.rados.0.smithi110.stdout:2162: expect (ObjNum 553 snap 168 seq_num 553) 2024-09-14T03:45:05.750 INFO:tasks.rados.rados.0.smithi110.stdout:2163: setattr oid 29 current snap is 216 2024-09-14T03:45:05.752 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 216 2024-09-14T03:45:05.752 INFO:tasks.rados.rados.0.smithi110.stdout:2164: seq_num 688 ranges {0=729088,729088=499712,1228800=704512} 2024-09-14T03:45:05.756 INFO:tasks.rados.rados.0.smithi110.stdout:2164: writing smithi11075562-50 from 0 to 729088 tid 1 2024-09-14T03:45:05.760 INFO:tasks.rados.rados.0.smithi110.stdout:2164: writing smithi11075562-50 from 729088 to 1228800 tid 2 2024-09-14T03:45:05.765 INFO:tasks.rados.rados.0.smithi110.stdout:2164: writing smithi11075562-50 from 1228800 to 1933312 tid 3 2024-09-14T03:45:05.765 INFO:tasks.rados.rados.0.smithi110.stdout:2161: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:45:05.765 INFO:tasks.rados.rados.0.smithi110.stdout:2161: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:45:05.765 INFO:tasks.rados.rados.0.smithi110.stdout:2161: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:45:05.765 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1100 (ObjNum 604 snap 186 seq_num 604) dirty exists 2024-09-14T03:45:05.766 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1248 (ObjNum 653 snap 202 seq_num 653) dirty exists 2024-09-14T03:45:05.766 INFO:tasks.rados.rados.0.smithi110.stdout:2161: done (3 left) 2024-09-14T03:45:05.766 INFO:tasks.rados.rados.0.smithi110.stdout:2163: done (2 left) 2024-09-14T03:45:05.766 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 216 2024-09-14T03:45:05.766 INFO:tasks.rados.rados.0.smithi110.stdout:2165: seq_num 689 ranges {4603904=745472,5349376=786432,6135808=417792} 2024-09-14T03:45:05.796 INFO:tasks.rados.rados.0.smithi110.stdout:2165: writing smithi11075562-23 from 4603904 to 5349376 tid 1 2024-09-14T03:45:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:2165: writing smithi11075562-23 from 5349376 to 6135808 tid 2 2024-09-14T03:45:05.803 INFO:tasks.rados.rados.0.smithi110.stdout:2165: writing smithi11075562-23 from 6135808 to 6553600 tid 3 2024-09-14T03:45:05.803 INFO:tasks.rados.rados.0.smithi110.stdout:2166: read oid 43 snap -1 2024-09-14T03:45:05.803 INFO:tasks.rados.rados.0.smithi110.stdout:2166: expect (ObjNum 679 snap 212 seq_num 679) 2024-09-14T03:45:05.803 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 216 2024-09-14T03:45:05.803 INFO:tasks.rados.rados.0.smithi110.stdout:2164: finishing write tid 1 to smithi11075562-50 2024-09-14T03:45:05.803 INFO:tasks.rados.rados.0.smithi110.stdout:2167: seq_num 690 ranges {2039808=761856,2801664=663552,3465216=409600,3874816=131072} 2024-09-14T03:45:05.818 INFO:tasks.rados.rados.0.smithi110.stdout:2167: writing smithi11075562-42 from 2039808 to 2801664 tid 1 2024-09-14T03:45:05.823 INFO:tasks.rados.rados.0.smithi110.stdout:2167: writing smithi11075562-42 from 2801664 to 3465216 tid 2 2024-09-14T03:45:05.824 INFO:tasks.rados.rados.0.smithi110.stdout:2167: writing smithi11075562-42 from 3465216 to 3874816 tid 3 2024-09-14T03:45:05.826 INFO:tasks.rados.rados.0.smithi110.stdout:2167: writing smithi11075562-42 from 3874816 to 4005888 tid 4 2024-09-14T03:45:05.826 INFO:tasks.rados.rados.0.smithi110.stdout:2168: read oid 29 snap -1 2024-09-14T03:45:05.826 INFO:tasks.rados.rados.0.smithi110.stdout:2168: expect (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:45:05.826 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 216 2024-09-14T03:45:05.827 INFO:tasks.rados.rados.0.smithi110.stdout:2164: finishing write tid 2 to smithi11075562-50 2024-09-14T03:45:05.827 INFO:tasks.rados.rados.0.smithi110.stdout:2164: finishing write tid 3 to smithi11075562-50 2024-09-14T03:45:05.827 INFO:tasks.rados.rados.0.smithi110.stdout:2164: finishing write tid 4 to smithi11075562-50 2024-09-14T03:45:05.827 INFO:tasks.rados.rados.0.smithi110.stdout:2169: seq_num 691 ranges {0=778240,778240=548864,1327104=614400,1941504=40960} 2024-09-14T03:45:05.830 INFO:tasks.rados.rados.0.smithi110.stdout:2169: writing smithi11075562-7 from 0 to 778240 tid 1 2024-09-14T03:45:05.834 INFO:tasks.rados.rados.0.smithi110.stdout:2169: writing smithi11075562-7 from 778240 to 1327104 tid 2 2024-09-14T03:45:05.838 INFO:tasks.rados.rados.0.smithi110.stdout:2169: writing smithi11075562-7 from 1327104 to 1941504 tid 3 2024-09-14T03:45:05.838 INFO:tasks.rados.rados.0.smithi110.stdout:2169: writing smithi11075562-7 from 1941504 to 1982464 tid 4 2024-09-14T03:45:05.838 INFO:tasks.rados.rados.0.smithi110.stdout:2170: rollback oid 26 current snap is 216 2024-09-14T03:45:05.839 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 213 2024-09-14T03:45:05.839 INFO:tasks.rados.rados.0.smithi110.stdout:2164: finishing write tid 5 to smithi11075562-50 2024-09-14T03:45:05.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 942 (ObjNum 688 snap 216 seq_num 688) dirty exists 2024-09-14T03:45:05.839 INFO:tasks.rados.rados.0.smithi110.stdout:2164: left oid 50 (ObjNum 688 snap 216 seq_num 688) 2024-09-14T03:45:05.839 INFO:tasks.rados.rados.0.smithi110.stdout:2164: done (7 left) 2024-09-14T03:45:05.839 INFO:tasks.rados.rados.0.smithi110.stdout:2171: delete oid 3 current snap is 216 2024-09-14T03:45:05.839 INFO:tasks.rados.rados.0.smithi110.stdout:2165: finishing write tid 1 to smithi11075562-23 2024-09-14T03:45:05.839 INFO:tasks.rados.rados.0.smithi110.stdout:2162: expect (ObjNum 553 snap 168 seq_num 553) 2024-09-14T03:45:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:05 smithi195 ceph-mon[82388]: Upgrade: Updating osd.6 2024-09-14T03:45:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:05 smithi195 ceph-mon[82388]: Deploying daemon osd.6 on smithi195 2024-09-14T03:45:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:05 smithi195 ceph-mon[82388]: osdmap e504: 8 total, 8 up, 8 in 2024-09-14T03:45:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:05 smithi110 ceph-mon[88262]: Upgrade: Updating osd.6 2024-09-14T03:45:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:05 smithi110 ceph-mon[88262]: Deploying daemon osd.6 on smithi195 2024-09-14T03:45:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:05 smithi110 ceph-mon[88262]: osdmap e504: 8 total, 8 up, 8 in 2024-09-14T03:45:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:05 smithi110 ceph-mon[86423]: Upgrade: Updating osd.6 2024-09-14T03:45:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:05 smithi110 ceph-mon[86423]: Deploying daemon osd.6 on smithi195 2024-09-14T03:45:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:05 smithi110 ceph-mon[86423]: osdmap e504: 8 total, 8 up, 8 in 2024-09-14T03:45:06.568 INFO:tasks.rados.rados.0.smithi110.stdout:2165: finishing write tid 2 to smithi11075562-23 2024-09-14T03:45:06.568 INFO:tasks.rados.rados.0.smithi110.stdout:2165: finishing write tid 3 to smithi11075562-23 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2165: finishing write tid 4 to smithi11075562-23 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2165: finishing write tid 5 to smithi11075562-23 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 946 (ObjNum 689 snap 216 seq_num 689) dirty exists 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2165: left oid 23 (ObjNum 689 snap 216 seq_num 689) 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2167: finishing write tid 1 to smithi11075562-42 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2167: finishing write tid 2 to smithi11075562-42 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2162: done (7 left) 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2165: done (6 left) 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2171: done (5 left) 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 216 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2167: finishing write tid 3 to smithi11075562-42 2024-09-14T03:45:06.569 INFO:tasks.rados.rados.0.smithi110.stdout:2167: finishing write tid 4 to smithi11075562-42 2024-09-14T03:45:06.570 INFO:tasks.rados.rados.0.smithi110.stdout:2167: finishing write tid 5 to smithi11075562-42 2024-09-14T03:45:06.570 INFO:tasks.rados.rados.0.smithi110.stdout:2167: finishing write tid 6 to smithi11075562-42 2024-09-14T03:45:06.570 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1296 (ObjNum 690 snap 216 seq_num 690) dirty exists 2024-09-14T03:45:06.570 INFO:tasks.rados.rados.0.smithi110.stdout:2167: left oid 42 (ObjNum 690 snap 216 seq_num 690) 2024-09-14T03:45:06.570 INFO:tasks.rados.rados.0.smithi110.stdout:2166: expect (ObjNum 679 snap 212 seq_num 679) 2024-09-14T03:45:06.755 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:06 smithi195 systemd[1]: Stopping Ceph osd.6 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:45:06.910 INFO:tasks.rados.rados.0.smithi110.stdout:2168: expect (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:45:06.945 INFO:tasks.rados.rados.0.smithi110.stdout:2169: finishing write tid 1 to smithi11075562-7 2024-09-14T03:45:06.945 INFO:tasks.rados.rados.0.smithi110.stdout:2169: finishing write tid 2 to smithi11075562-7 2024-09-14T03:45:06.945 INFO:tasks.rados.rados.0.smithi110.stdout:2170: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:2170: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 787 (ObjNum 97 snap 0 seq_num 1497030638) dirty dne 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:2169: finishing write tid 3 to smithi11075562-7 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:2169: finishing write tid 4 to smithi11075562-7 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:2169: finishing write tid 5 to smithi11075562-7 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:2169: finishing write tid 6 to smithi11075562-7 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1253 (ObjNum 691 snap 216 seq_num 691) dirty exists 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:2169: left oid 7 (ObjNum 691 snap 216 seq_num 691) 2024-09-14T03:45:06.946 INFO:tasks.rados.rados.0.smithi110.stdout:2172: seq_num 693 ranges {0=409600,409600=770048,1179648=581632,1761280=253952} 2024-09-14T03:45:06.947 INFO:tasks.rados.rados.0.smithi110.stdout:2172: writing smithi11075562-40 from 0 to 409600 tid 1 2024-09-14T03:45:06.952 INFO:tasks.rados.rados.0.smithi110.stdout:2172: writing smithi11075562-40 from 409600 to 1179648 tid 2 2024-09-14T03:45:06.955 INFO:tasks.rados.rados.0.smithi110.stdout:2172: writing smithi11075562-40 from 1179648 to 1761280 tid 3 2024-09-14T03:45:06.957 INFO:tasks.rados.rados.0.smithi110.stdout:2172: writing smithi11075562-40 from 1761280 to 2015232 tid 4 2024-09-14T03:45:06.957 INFO:tasks.rados.rados.0.smithi110.stdout:2166: done (5 left) 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:2167: done (4 left) 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:2168: done (3 left) 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:2169: done (2 left) 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:2170: done (1 left) 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:2173: rollback oid 20 current snap is 216 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 206 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:2172: finishing write tid 1 to smithi11075562-40 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:2172: finishing write tid 2 to smithi11075562-40 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 216 2024-09-14T03:45:06.958 INFO:tasks.rados.rados.0.smithi110.stdout:2174: seq_num 694 ranges {524288=425984,950272=655360,1605632=786432,2392064=163840} 2024-09-14T03:45:06.962 INFO:tasks.rados.rados.0.smithi110.stdout:2174: writing smithi11075562-11 from 524288 to 950272 tid 1 2024-09-14T03:45:06.966 INFO:tasks.rados.rados.0.smithi110.stdout:2174: writing smithi11075562-11 from 950272 to 1605632 tid 2 2024-09-14T03:45:06.970 INFO:tasks.rados.rados.0.smithi110.stdout:2174: writing smithi11075562-11 from 1605632 to 2392064 tid 3 2024-09-14T03:45:06.972 INFO:tasks.rados.rados.0.smithi110.stdout:2174: writing smithi11075562-11 from 2392064 to 2555904 tid 4 2024-09-14T03:45:06.972 INFO:tasks.rados.rados.0.smithi110.stdout:2175: rollback oid 45 current snap is 216 2024-09-14T03:45:06.972 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 210 2024-09-14T03:45:06.972 INFO:tasks.rados.rados.0.smithi110.stdout:2172: finishing write tid 3 to smithi11075562-40 2024-09-14T03:45:06.972 INFO:tasks.rados.rados.0.smithi110.stdout:2173: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:45:06.972 INFO:tasks.rados.rados.0.smithi110.stdout:2172: finishing write tid 4 to smithi11075562-40 2024-09-14T03:45:06.972 INFO:tasks.rados.rados.0.smithi110.stdout:2172: finishing write tid 5 to smithi11075562-40 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2173: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2172: finishing write tid 6 to smithi11075562-40 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1178 (ObjNum 693 snap 216 seq_num 693) dirty exists 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2172: left oid 40 (ObjNum 693 snap 216 seq_num 693) 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2173: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1460 (ObjNum 663 snap 204 seq_num 663) dirty exists 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2174: finishing write tid 1 to smithi11075562-11 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2174: finishing write tid 2 to smithi11075562-11 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2172: done (3 left) 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2173: done (2 left) 2024-09-14T03:45:06.973 INFO:tasks.rados.rados.0.smithi110.stdout:2176: snap_create 2024-09-14T03:45:06.976 INFO:tasks.rados.rados.0.smithi110.stdout:2174: finishing write tid 3 to smithi11075562-11 2024-09-14T03:45:06.976 INFO:tasks.rados.rados.0.smithi110.stdout:2174: finishing write tid 4 to smithi11075562-11 2024-09-14T03:45:06.976 INFO:tasks.rados.rados.0.smithi110.stdout:2174: finishing write tid 5 to smithi11075562-11 2024-09-14T03:45:06.976 INFO:tasks.rados.rados.0.smithi110.stdout:2175: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:45:06.976 INFO:tasks.rados.rados.0.smithi110.stdout:2175: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:45:06.977 INFO:tasks.rados.rados.0.smithi110.stdout:2175: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:45:06.977 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 1562 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:45:06.977 INFO:tasks.rados.rados.0.smithi110.stdout:2174: finishing write tid 6 to smithi11075562-11 2024-09-14T03:45:06.977 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1105 (ObjNum 694 snap 216 seq_num 694) dirty exists 2024-09-14T03:45:06.977 INFO:tasks.rados.rados.0.smithi110.stdout:2174: left oid 11 (ObjNum 694 snap 216 seq_num 694) 2024-09-14T03:45:07.158 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:06 smithi195 ceph-mon[82388]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T03:45:07.158 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:06 smithi195 ceph-mon[82388]: pgmap v386: 121 pgs: 121 active+clean; 227 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-14T03:45:07.158 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[47115]: 2024-09-14T03:45:06.947+0000 7faf8f4c5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:45:07.158 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[47115]: 2024-09-14T03:45:06.947+0000 7faf8f4c5640 -1 osd.6 505 *** Got signal Terminated *** 2024-09-14T03:45:07.159 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[47115]: 2024-09-14T03:45:06.947+0000 7faf8f4c5640 -1 osd.6 505 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:45:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:06 smithi110 ceph-mon[88262]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T03:45:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:06 smithi110 ceph-mon[88262]: pgmap v386: 121 pgs: 121 active+clean; 227 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-14T03:45:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:06 smithi110 ceph-mon[86423]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T03:45:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:06 smithi110 ceph-mon[86423]: pgmap v386: 121 pgs: 121 active+clean; 227 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-14T03:45:07.765 INFO:tasks.rados.rados.0.smithi110.stdout:2174: done (2 left) 2024-09-14T03:45:07.765 INFO:tasks.rados.rados.0.smithi110.stdout:2175: done (1 left) 2024-09-14T03:45:07.765 INFO:tasks.rados.rados.0.smithi110.stdout:2176: done (0 left) 2024-09-14T03:45:07.766 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 217 2024-09-14T03:45:07.766 INFO:tasks.rados.rados.0.smithi110.stdout:2177: seq_num 695 ranges {1933312=442368,2375680=540672,2916352=598016,3514368=466944} 2024-09-14T03:45:07.778 INFO:tasks.rados.rados.0.smithi110.stdout:2177: writing smithi11075562-50 from 1933312 to 2375680 tid 1 2024-09-14T03:45:07.781 INFO:tasks.rados.rados.0.smithi110.stdout:2177: writing smithi11075562-50 from 2375680 to 2916352 tid 2 2024-09-14T03:45:07.785 INFO:tasks.rados.rados.0.smithi110.stdout:2177: writing smithi11075562-50 from 2916352 to 3514368 tid 3 2024-09-14T03:45:07.788 INFO:tasks.rados.rados.0.smithi110.stdout:2177: writing smithi11075562-50 from 3514368 to 3981312 tid 4 2024-09-14T03:45:07.789 INFO:tasks.rados.rados.0.smithi110.stdout:2178: read oid 28 snap 210 2024-09-14T03:45:07.789 INFO:tasks.rados.rados.0.smithi110.stdout:2178: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:45:07.789 INFO:tasks.rados.rados.0.smithi110.stdout:2179: copy_from oid 5 from oid 49 current snap is 217 2024-09-14T03:45:07.789 INFO:tasks.rados.rados.0.smithi110.stdout:2177: finishing write tid 1 to smithi11075562-50 2024-09-14T03:45:07.789 INFO:tasks.rados.rados.0.smithi110.stdout:2177: finishing write tid 2 to smithi11075562-50 2024-09-14T03:45:07.789 INFO:tasks.rados.rados.0.smithi110.stdout:2177: finishing write tid 3 to smithi11075562-50 2024-09-14T03:45:07.789 INFO:tasks.rados.rados.0.smithi110.stdout:2180: snap_remove snap 212 2024-09-14T03:45:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:07 smithi110 ceph-mon[88262]: osd.6 marked itself down and dead 2024-09-14T03:45:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:07 smithi110 ceph-mon[86423]: osd.6 marked itself down and dead 2024-09-14T03:45:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:07 smithi195 ceph-mon[82388]: osd.6 marked itself down and dead 2024-09-14T03:45:08.549 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:08 smithi195 podman[95774]: 2024-09-14 03:45:08.280735771 +0000 UTC m=+1.447415973 container died f979f8dd4ea0581e5fdb458563a03565cdce6b0ef4d6078c316448366d53d1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-14T03:45:08.730 INFO:tasks.rados.rados.0.smithi110.stdout:2180: done (3 left) 2024-09-14T03:45:08.731 INFO:tasks.rados.rados.0.smithi110.stdout:2181: read oid 35 snap 196 2024-09-14T03:45:08.731 INFO:tasks.rados.rados.0.smithi110.stdout:2181: expect deleted 2024-09-14T03:45:08.731 INFO:tasks.rados.rados.0.smithi110.stdout:2182: rollback oid 11 current snap is 217 2024-09-14T03:45:08.731 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 204 2024-09-14T03:45:08.738 INFO:tasks.rados.rados.0.smithi110.stdout:2177: finishing write tid 4 to smithi11075562-50 2024-09-14T03:45:08.738 INFO:tasks.rados.rados.0.smithi110.stdout:2177: finishing write tid 5 to smithi11075562-50 2024-09-14T03:45:08.738 INFO:tasks.rados.rados.0.smithi110.stdout:2178: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:45:08.753 INFO:tasks.rados.rados.0.smithi110.stdout:2177: finishing write tid 6 to smithi11075562-50 2024-09-14T03:45:08.753 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 947 (ObjNum 695 snap 217 seq_num 695) dirty exists 2024-09-14T03:45:08.753 INFO:tasks.rados.rados.0.smithi110.stdout:2177: left oid 50 (ObjNum 695 snap 217 seq_num 695) 2024-09-14T03:45:08.753 INFO:tasks.rados.rados.0.smithi110.stdout:2177: done (4 left) 2024-09-14T03:45:08.753 INFO:tasks.rados.rados.0.smithi110.stdout:2178: done (3 left) 2024-09-14T03:45:08.753 INFO:tasks.rados.rados.0.smithi110.stdout:2183: read oid 18 snap 213 2024-09-14T03:45:08.754 INFO:tasks.rados.rados.0.smithi110.stdout:2183: expect (ObjNum 682 snap 213 seq_num 682) 2024-09-14T03:45:08.754 INFO:tasks.rados.rados.0.smithi110.stdout:2184: snap_create 2024-09-14T03:45:08.754 INFO:tasks.rados.rados.0.smithi110.stdout:2182: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:45:08.754 INFO:tasks.rados.rados.0.smithi110.stdout:2182: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:45:08.754 INFO:tasks.rados.rados.0.smithi110.stdout:2182: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:45:08.754 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1108 (ObjNum 604 snap 186 seq_num 604) dirty exists 2024-09-14T03:45:08.757 INFO:tasks.rados.rados.0.smithi110.stdout:2179: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:45:08.757 INFO:tasks.rados.rados.0.smithi110.stdout:2179: finishing copy_from to smithi11075562-5 2024-09-14T03:45:08.758 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1256 (ObjNum 660 snap 204 seq_num 660) dirty exists 2024-09-14T03:45:08.763 INFO:tasks.rados.rados.0.smithi110.stdout:2183: expect (ObjNum 682 snap 213 seq_num 682) 2024-09-14T03:45:08.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:08 smithi195 ceph-mon[82388]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:45:08.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:08 smithi195 ceph-mon[82388]: osdmap e506: 8 total, 7 up, 8 in 2024-09-14T03:45:08.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:08 smithi195 ceph-mon[82388]: pgmap v388: 121 pgs: 14 stale+active+clean, 107 active+clean; 247 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 6.5 MiB/s wr, 22 op/s 2024-09-14T03:45:08.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:08 smithi195 ceph-mon[82388]: osdmap e507: 8 total, 7 up, 8 in 2024-09-14T03:45:08.925 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:08 smithi195 podman[95774]: 2024-09-14 03:45:08.647361995 +0000 UTC m=+1.814042229 container cleanup f979f8dd4ea0581e5fdb458563a03565cdce6b0ef4d6078c316448366d53d1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-14T03:45:08.925 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:08 smithi195 bash[95774]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6 2024-09-14T03:45:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:08 smithi110 ceph-mon[86423]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:45:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:08 smithi110 ceph-mon[86423]: osdmap e506: 8 total, 7 up, 8 in 2024-09-14T03:45:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:08 smithi110 ceph-mon[86423]: pgmap v388: 121 pgs: 14 stale+active+clean, 107 active+clean; 247 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 6.5 MiB/s wr, 22 op/s 2024-09-14T03:45:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:08 smithi110 ceph-mon[86423]: osdmap e507: 8 total, 7 up, 8 in 2024-09-14T03:45:09.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:08 smithi110 ceph-mon[88262]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:45:09.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:08 smithi110 ceph-mon[88262]: osdmap e506: 8 total, 7 up, 8 in 2024-09-14T03:45:09.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:08 smithi110 ceph-mon[88262]: pgmap v388: 121 pgs: 14 stale+active+clean, 107 active+clean; 247 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 6.5 MiB/s wr, 22 op/s 2024-09-14T03:45:09.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:08 smithi110 ceph-mon[88262]: osdmap e507: 8 total, 7 up, 8 in 2024-09-14T03:45:09.208 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:08 smithi195 podman[95800]: 2024-09-14 03:45:08.923677685 +0000 UTC m=+0.639582653 container remove f979f8dd4ea0581e5fdb458563a03565cdce6b0ef4d6078c316448366d53d1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T03:45:09.569 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:09 smithi195 podman[95890]: 2024-09-14 03:45:09.257929841 +0000 UTC m=+0.103396082 container create 025af3b7644c20acab6d46d511eebba16e9a9a3176a2b2a4c703d674be283609 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T03:45:09.569 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:09 smithi195 podman[95890]: 2024-09-14 03:45:09.170555534 +0000 UTC m=+0.016021772 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:45:09.569 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:09 smithi195 podman[95890]: 2024-09-14 03:45:09.468865537 +0000 UTC m=+0.314331778 container init 025af3b7644c20acab6d46d511eebba16e9a9a3176a2b2a4c703d674be283609 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-14T03:45:09.570 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:09 smithi195 podman[95890]: 2024-09-14 03:45:09.472235822 +0000 UTC m=+0.317702069 container start 025af3b7644c20acab6d46d511eebba16e9a9a3176a2b2a4c703d674be283609 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-14T03:45:09.570 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:09 smithi195 podman[95890]: 2024-09-14 03:45:09.568544858 +0000 UTC m=+0.414011131 container attach 025af3b7644c20acab6d46d511eebba16e9a9a3176a2b2a4c703d674be283609 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-14T03:45:09.730 INFO:tasks.rados.rados.0.smithi110.stdout:2179: done (4 left) 2024-09-14T03:45:09.730 INFO:tasks.rados.rados.0.smithi110.stdout:2181: done (3 left) 2024-09-14T03:45:09.731 INFO:tasks.rados.rados.0.smithi110.stdout:2182: done (2 left) 2024-09-14T03:45:09.731 INFO:tasks.rados.rados.0.smithi110.stdout:2183: done (1 left) 2024-09-14T03:45:09.731 INFO:tasks.rados.rados.0.smithi110.stdout:2184: done (0 left) 2024-09-14T03:45:09.731 INFO:tasks.rados.rados.0.smithi110.stdout:2185: snap_create 2024-09-14T03:45:09.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:09 smithi195 podman[95890]: 2024-09-14 03:45:09.661890901 +0000 UTC m=+0.507357141 container died 025af3b7644c20acab6d46d511eebba16e9a9a3176a2b2a4c703d674be283609 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-14T03:45:10.289 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:45:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:45:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:45:10.709 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 podman[95913]: 2024-09-14 03:45:10.289143018 +0000 UTC m=+0.620872396 container remove 025af3b7644c20acab6d46d511eebba16e9a9a3176a2b2a4c703d674be283609 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231) 2024-09-14T03:45:10.709 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-14T03:45:10.709 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 systemd[1]: Stopped Ceph osd.6 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:45:10.709 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.6.service: Consumed 13.936s CPU time. 2024-09-14T03:45:10.709 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 systemd[1]: Starting Ceph osd.6 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:45:10.736 INFO:tasks.rados.rados.0.smithi110.stdout:2185: done (0 left) 2024-09-14T03:45:10.736 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 219 2024-09-14T03:45:10.736 INFO:tasks.rados.rados.0.smithi110.stdout:2186: seq_num 696 ranges {3981312=458752,4440064=417792,4857856=753664,5611520=434176} 2024-09-14T03:45:10.759 INFO:tasks.rados.rados.0.smithi110.stdout:2186: writing smithi11075562-50 from 3981312 to 4440064 tid 1 2024-09-14T03:45:10.761 INFO:tasks.rados.rados.0.smithi110.stdout:2186: writing smithi11075562-50 from 4440064 to 4857856 tid 2 2024-09-14T03:45:10.767 INFO:tasks.rados.rados.0.smithi110.stdout:2186: writing smithi11075562-50 from 4857856 to 5611520 tid 3 2024-09-14T03:45:10.770 INFO:tasks.rados.rados.0.smithi110.stdout:2186: writing smithi11075562-50 from 5611520 to 6045696 tid 4 2024-09-14T03:45:10.770 INFO:tasks.rados.rados.0.smithi110.stdout:2187: rmattr oid 14 current snap is 219 2024-09-14T03:45:10.770 INFO:tasks.rados.rados.0.smithi110.stdout:2186: finishing write tid 1 to smithi11075562-50 2024-09-14T03:45:10.770 INFO:tasks.rados.rados.0.smithi110.stdout:2186: finishing write tid 2 to smithi11075562-50 2024-09-14T03:45:10.770 INFO:tasks.rados.rados.0.smithi110.stdout:2188: read oid 45 snap -1 2024-09-14T03:45:10.770 INFO:tasks.rados.rados.0.smithi110.stdout:2188: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:45:10.770 INFO:tasks.rados.rados.0.smithi110.stdout:2189: snap_create 2024-09-14T03:45:10.772 INFO:tasks.rados.rados.0.smithi110.stdout:2186: finishing write tid 3 to smithi11075562-50 2024-09-14T03:45:10.773 INFO:tasks.rados.rados.0.smithi110.stdout:2186: finishing write tid 4 to smithi11075562-50 2024-09-14T03:45:10.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1482 (ObjNum 542 snap 166 seq_num 542) dirty exists 2024-09-14T03:45:10.773 INFO:tasks.rados.rados.0.smithi110.stdout:2186: finishing write tid 5 to smithi11075562-50 2024-09-14T03:45:10.774 INFO:tasks.rados.rados.0.smithi110.stdout:2186: finishing write tid 6 to smithi11075562-50 2024-09-14T03:45:10.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 952 (ObjNum 696 snap 219 seq_num 696) dirty exists 2024-09-14T03:45:10.775 INFO:tasks.rados.rados.0.smithi110.stdout:2186: left oid 50 (ObjNum 696 snap 219 seq_num 696) 2024-09-14T03:45:10.782 INFO:tasks.rados.rados.0.smithi110.stdout:2188: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:45:10.993 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:10 smithi195 ceph-mon[82388]: osdmap e508: 8 total, 7 up, 8 in 2024-09-14T03:45:10.994 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:10 smithi195 ceph-mon[82388]: pgmap v391: 121 pgs: 5 active+undersized, 11 stale+active+clean, 2 active+undersized+degraded, 103 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 6.2 MiB/s wr, 21 op/s; 11/429 objects degraded (2.564%) 2024-09-14T03:45:10.994 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 podman[96013]: 2024-09-14 03:45:10.62741361 +0000 UTC m=+0.015943277 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:45:10.994 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 podman[96013]: 2024-09-14 03:45:10.741802645 +0000 UTC m=+0.130332308 container create 04d74139341cdb1008ce8c7ced4952d8e5f44640bb9b25607a1aaab9b7238821 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:45:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:10 smithi110 ceph-mon[88262]: osdmap e508: 8 total, 7 up, 8 in 2024-09-14T03:45:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:10 smithi110 ceph-mon[88262]: pgmap v391: 121 pgs: 5 active+undersized, 11 stale+active+clean, 2 active+undersized+degraded, 103 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 6.2 MiB/s wr, 21 op/s; 11/429 objects degraded (2.564%) 2024-09-14T03:45:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:10 smithi110 ceph-mon[86423]: osdmap e508: 8 total, 7 up, 8 in 2024-09-14T03:45:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:10 smithi110 ceph-mon[86423]: pgmap v391: 121 pgs: 5 active+undersized, 11 stale+active+clean, 2 active+undersized+degraded, 103 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 6.2 MiB/s wr, 21 op/s; 11/429 objects degraded (2.564%) 2024-09-14T03:45:11.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 podman[96013]: 2024-09-14 03:45:10.993023473 +0000 UTC m=+0.381553140 container init 04d74139341cdb1008ce8c7ced4952d8e5f44640bb9b25607a1aaab9b7238821 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-14T03:45:11.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:10 smithi195 podman[96013]: 2024-09-14 03:45:10.996760144 +0000 UTC m=+0.385289808 container start 04d74139341cdb1008ce8c7ced4952d8e5f44640bb9b25607a1aaab9b7238821 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-14T03:45:11.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 podman[96013]: 2024-09-14 03:45:11.076284869 +0000 UTC m=+0.464814553 container attach 04d74139341cdb1008ce8c7ced4952d8e5f44640bb9b25607a1aaab9b7238821 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:45:11.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:45:11.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:45:11.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:45:11.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2186: done (3 left) 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2187: done (2 left) 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2188: done (1 left) 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2189: done (0 left) 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2190: read oid 48 snap -1 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2190: expect (ObjNum 635 snap 196 seq_num 635) 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2191: read oid 22 snap -1 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2191: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 220 2024-09-14T03:45:11.741 INFO:tasks.rados.rados.0.smithi110.stdout:2192: seq_num 697 ranges {2342912=475136,2818048=704512,3522560=557056,4079616=344064} 2024-09-14T03:45:11.757 INFO:tasks.rados.rados.0.smithi110.stdout:2192: writing smithi11075562-5 from 2342912 to 2818048 tid 1 2024-09-14T03:45:11.761 INFO:tasks.rados.rados.0.smithi110.stdout:2192: writing smithi11075562-5 from 2818048 to 3522560 tid 2 2024-09-14T03:45:11.765 INFO:tasks.rados.rados.0.smithi110.stdout:2192: writing smithi11075562-5 from 3522560 to 4079616 tid 3 2024-09-14T03:45:11.767 INFO:tasks.rados.rados.0.smithi110.stdout:2192: writing smithi11075562-5 from 4079616 to 4423680 tid 4 2024-09-14T03:45:11.767 INFO:tasks.rados.rados.0.smithi110.stdout:2193: rollback oid 15 current snap is 220 2024-09-14T03:45:11.767 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 219 2024-09-14T03:45:11.768 INFO:tasks.rados.rados.0.smithi110.stdout:2190: expect (ObjNum 635 snap 196 seq_num 635) 2024-09-14T03:45:11.801 INFO:tasks.rados.rados.0.smithi110.stdout:2192: finishing write tid 1 to smithi11075562-5 2024-09-14T03:45:11.801 INFO:tasks.rados.rados.0.smithi110.stdout:2192: finishing write tid 2 to smithi11075562-5 2024-09-14T03:45:11.801 INFO:tasks.rados.rados.0.smithi110.stdout:2192: finishing write tid 3 to smithi11075562-5 2024-09-14T03:45:11.801 INFO:tasks.rados.rados.0.smithi110.stdout:2192: finishing write tid 4 to smithi11075562-5 2024-09-14T03:45:11.801 INFO:tasks.rados.rados.0.smithi110.stdout:2192: finishing write tid 5 to smithi11075562-5 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2192: finishing write tid 6 to smithi11075562-5 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1570 (ObjNum 697 snap 220 seq_num 697) dirty exists 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2192: left oid 5 (ObjNum 697 snap 220 seq_num 697) 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2190: done (3 left) 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2192: done (2 left) 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2194: read oid 35 snap -1 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2194: expect deleted 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 220 2024-09-14T03:45:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2195: seq_num 698 ranges {1040384=499712,1540096=589824,2129920=770048,2899968=245760} 2024-09-14T03:45:11.809 INFO:tasks.rados.rados.0.smithi110.stdout:2195: writing smithi11075562-48 from 1040384 to 1540096 tid 1 2024-09-14T03:45:11.813 INFO:tasks.rados.rados.0.smithi110.stdout:2195: writing smithi11075562-48 from 1540096 to 2129920 tid 2 2024-09-14T03:45:11.817 INFO:tasks.rados.rados.0.smithi110.stdout:2195: writing smithi11075562-48 from 2129920 to 2899968 tid 3 2024-09-14T03:45:11.819 INFO:tasks.rados.rados.0.smithi110.stdout:2195: writing smithi11075562-48 from 2899968 to 3145728 tid 4 2024-09-14T03:45:11.819 INFO:tasks.rados.rados.0.smithi110.stdout:2196: snap_create 2024-09-14T03:45:11.819 INFO:tasks.rados.rados.0.smithi110.stdout:2191: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:45:11.882 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-mon[82388]: Health check failed: Degraded data redundancy: 11/429 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:11.882 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-mon[82388]: osdmap e509: 8 total, 7 up, 8 in 2024-09-14T03:45:11.882 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-14T03:45:11.882 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-14T03:45:11.882 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T03:45:11.882 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T03:45:11.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:45:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:45:11] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.51.0" 2024-09-14T03:45:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:11 smithi110 ceph-mon[86423]: Health check failed: Degraded data redundancy: 11/429 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:11 smithi110 ceph-mon[86423]: osdmap e509: 8 total, 7 up, 8 in 2024-09-14T03:45:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:11 smithi110 ceph-mon[88262]: Health check failed: Degraded data redundancy: 11/429 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:11 smithi110 ceph-mon[88262]: osdmap e509: 8 total, 7 up, 8 in 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate[96028]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-14T03:45:12.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 bash[96013]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-14T03:45:12.210 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:11 smithi195 podman[96013]: 2024-09-14 03:45:11.908855825 +0000 UTC m=+1.297385497 container died 04d74139341cdb1008ce8c7ced4952d8e5f44640bb9b25607a1aaab9b7238821 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-09-14T03:45:12.913 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:12 smithi195 ceph-mon[82388]: osdmap e510: 8 total, 7 up, 8 in 2024-09-14T03:45:12.913 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:12 smithi195 ceph-mon[82388]: pgmap v394: 121 pgs: 21 active+undersized, 3 stale+active+clean, 3 active+undersized+degraded, 94 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.5 MiB/s wr, 10 op/s; 28/438 objects degraded (6.393%) 2024-09-14T03:45:12.913 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:12 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:12.913 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:12.914 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:12 smithi195 podman[96153]: 2024-09-14 03:45:12.563738538 +0000 UTC m=+0.644242043 container remove 04d74139341cdb1008ce8c7ced4952d8e5f44640bb9b25607a1aaab9b7238821 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231) 2024-09-14T03:45:12.914 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:12 smithi195 podman[96215]: 2024-09-14 03:45:12.675375362 +0000 UTC m=+0.015772207 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:45:12.914 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:12 smithi195 podman[96215]: 2024-09-14 03:45:12.772868711 +0000 UTC m=+0.113265549 container create 5305ca81bc266a4ffa06865f4a20ba73c0f10b9f058b99f246872889c8f4b7cf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T03:45:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:12 smithi110 ceph-mon[88262]: osdmap e510: 8 total, 7 up, 8 in 2024-09-14T03:45:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:12 smithi110 ceph-mon[88262]: pgmap v394: 121 pgs: 21 active+undersized, 3 stale+active+clean, 3 active+undersized+degraded, 94 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.5 MiB/s wr, 10 op/s; 28/438 objects degraded (6.393%) 2024-09-14T03:45:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:12 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:12 smithi110 ceph-mon[86423]: osdmap e510: 8 total, 7 up, 8 in 2024-09-14T03:45:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:12 smithi110 ceph-mon[86423]: pgmap v394: 121 pgs: 21 active+undersized, 3 stale+active+clean, 3 active+undersized+degraded, 94 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.5 MiB/s wr, 10 op/s; 28/438 objects degraded (6.393%) 2024-09-14T03:45:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:12 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:13.257 INFO:tasks.rados.rados.0.smithi110.stdout:2193: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:45:13.257 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1244 (ObjNum 65 snap 0 seq_num 1903449458) dirty dne 2024-09-14T03:45:13.257 INFO:tasks.rados.rados.0.smithi110.stdout:2195: finishing write tid 1 to smithi11075562-48 2024-09-14T03:45:13.257 INFO:tasks.rados.rados.0.smithi110.stdout:2195: finishing write tid 2 to smithi11075562-48 2024-09-14T03:45:13.257 INFO:tasks.rados.rados.0.smithi110.stdout:2191: done (4 left) 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2193: done (3 left) 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2194: done (2 left) 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2196: done (1 left) 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2197: snap_create 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2195: finishing write tid 3 to smithi11075562-48 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2195: finishing write tid 4 to smithi11075562-48 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2195: finishing write tid 5 to smithi11075562-48 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2195: finishing write tid 6 to smithi11075562-48 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1113 (ObjNum 698 snap 220 seq_num 698) dirty exists 2024-09-14T03:45:13.258 INFO:tasks.rados.rados.0.smithi110.stdout:2195: left oid 48 (ObjNum 698 snap 220 seq_num 698) 2024-09-14T03:45:13.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:13 smithi195 podman[96215]: 2024-09-14 03:45:13.042009051 +0000 UTC m=+0.382405891 container init 5305ca81bc266a4ffa06865f4a20ba73c0f10b9f058b99f246872889c8f4b7cf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:45:13.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:13 smithi195 podman[96215]: 2024-09-14 03:45:13.045126457 +0000 UTC m=+0.385523306 container start 5305ca81bc266a4ffa06865f4a20ba73c0f10b9f058b99f246872889c8f4b7cf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T03:45:13.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:13 smithi195 ceph-osd[96234]: -- 172.21.15.195:0/3347092026 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a4e20bda40 con 0x55a4e209bc00 2024-09-14T03:45:13.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:13 smithi195 bash[96215]: 5305ca81bc266a4ffa06865f4a20ba73c0f10b9f058b99f246872889c8f4b7cf 2024-09-14T03:45:13.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:13 smithi195 systemd[1]: Started Ceph osd.6 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:45:13.729 INFO:tasks.rados.rados.0.smithi110.stdout:2195: done (1 left) 2024-09-14T03:45:13.730 INFO:tasks.rados.rados.0.smithi110.stdout:2197: done (0 left) 2024-09-14T03:45:13.730 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 222 2024-09-14T03:45:13.730 INFO:tasks.rados.rados.0.smithi110.stdout:2198: seq_num 699 ranges {524288=516096,1040384=475136,1515520=573440,2088960=557056} 2024-09-14T03:45:13.735 INFO:tasks.rados.rados.0.smithi110.stdout:2198: writing smithi11075562-11 from 524288 to 1040384 tid 1 2024-09-14T03:45:13.737 INFO:tasks.rados.rados.0.smithi110.stdout:2198: writing smithi11075562-11 from 1040384 to 1515520 tid 2 2024-09-14T03:45:13.741 INFO:tasks.rados.rados.0.smithi110.stdout:2198: writing smithi11075562-11 from 1515520 to 2088960 tid 3 2024-09-14T03:45:13.745 INFO:tasks.rados.rados.0.smithi110.stdout:2198: writing smithi11075562-11 from 2088960 to 2646016 tid 4 2024-09-14T03:45:13.745 INFO:tasks.rados.rados.0.smithi110.stdout:2199: delete oid 48 current snap is 222 2024-09-14T03:45:13.745 INFO:tasks.rados.rados.0.smithi110.stdout:2198: finishing write tid 1 to smithi11075562-11 2024-09-14T03:45:13.745 INFO:tasks.rados.rados.0.smithi110.stdout:2198: finishing write tid 2 to smithi11075562-11 2024-09-14T03:45:13.746 INFO:tasks.rados.rados.0.smithi110.stdout:2198: finishing write tid 3 to smithi11075562-11 2024-09-14T03:45:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2198: finishing write tid 4 to smithi11075562-11 2024-09-14T03:45:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2198: finishing write tid 5 to smithi11075562-11 2024-09-14T03:45:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2199: done (1 left) 2024-09-14T03:45:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2200: snap_create 2024-09-14T03:45:13.749 INFO:tasks.rados.rados.0.smithi110.stdout:2198: finishing write tid 6 to smithi11075562-11 2024-09-14T03:45:13.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1118 (ObjNum 699 snap 222 seq_num 699) dirty exists 2024-09-14T03:45:13.749 INFO:tasks.rados.rados.0.smithi110.stdout:2198: left oid 11 (ObjNum 699 snap 222 seq_num 699) 2024-09-14T03:45:14.035 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:13 smithi195 ceph-mon[82388]: osdmap e511: 8 total, 7 up, 8 in 2024-09-14T03:45:14.035 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:14.035 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:14.035 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:14.035 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:13 smithi195 ceph-mon[82388]: osdmap e512: 8 total, 7 up, 8 in 2024-09-14T03:45:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[86423]: osdmap e511: 8 total, 7 up, 8 in 2024-09-14T03:45:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[86423]: osdmap e512: 8 total, 7 up, 8 in 2024-09-14T03:45:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[88262]: osdmap e511: 8 total, 7 up, 8 in 2024-09-14T03:45:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:13 smithi110 ceph-mon[88262]: osdmap e512: 8 total, 7 up, 8 in 2024-09-14T03:45:14.747 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:45:14.689+0000 7f7695741640 -1 Falling back to public interface 2024-09-14T03:45:14.751 INFO:tasks.rados.rados.0.smithi110.stdout:2198: done (1 left) 2024-09-14T03:45:14.751 INFO:tasks.rados.rados.0.smithi110.stdout:2200: done (0 left) 2024-09-14T03:45:14.751 INFO:tasks.rados.rados.0.smithi110.stdout:2201: read oid 1 snap -1 2024-09-14T03:45:14.751 INFO:tasks.rados.rados.0.smithi110.stdout:2201: expect (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:45:14.752 INFO:tasks.rados.rados.0.smithi110.stdout:2202: read oid 17 snap -1 2024-09-14T03:45:14.752 INFO:tasks.rados.rados.0.smithi110.stdout:2202: expect (ObjNum 668 snap 206 seq_num 668) 2024-09-14T03:45:14.752 INFO:tasks.rados.rados.0.smithi110.stdout:2203: snap_remove snap 221 2024-09-14T03:45:15.051 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:14 smithi195 ceph-mon[82388]: pgmap v397: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 7 op/s; 47/441 objects degraded (10.658%) 2024-09-14T03:45:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:14 smithi110 ceph-mon[88262]: pgmap v397: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 7 op/s; 47/441 objects degraded (10.658%) 2024-09-14T03:45:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:14 smithi110 ceph-mon[86423]: pgmap v397: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 7 op/s; 47/441 objects degraded (10.658%) 2024-09-14T03:45:15.756 INFO:tasks.rados.rados.0.smithi110.stdout:2203: done (2 left) 2024-09-14T03:45:15.756 INFO:tasks.rados.rados.0.smithi110.stdout:2204: rollback oid 11 current snap is 223 2024-09-14T03:45:15.756 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 213 2024-09-14T03:45:15.756 INFO:tasks.rados.rados.0.smithi110.stdout:2201: expect (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:45:15.820 INFO:tasks.rados.rados.0.smithi110.stdout:2202: expect (ObjNum 668 snap 206 seq_num 668) 2024-09-14T03:45:15.943 INFO:tasks.rados.rados.0.smithi110.stdout:2204: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:45:15.943 INFO:tasks.rados.rados.0.smithi110.stdout:2204: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:45:15.943 INFO:tasks.rados.rados.0.smithi110.stdout:2201: done (2 left) 2024-09-14T03:45:15.944 INFO:tasks.rados.rados.0.smithi110.stdout:2202: done (1 left) 2024-09-14T03:45:15.944 INFO:tasks.rados.rados.0.smithi110.stdout:2205: snap_create 2024-09-14T03:45:15.944 INFO:tasks.rados.rados.0.smithi110.stdout:2204: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:45:15.944 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1122 (ObjNum 604 snap 186 seq_num 604) dirty exists 2024-09-14T03:45:15.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:15 smithi195 ceph-mon[82388]: osdmap e513: 8 total, 7 up, 8 in 2024-09-14T03:45:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:15 smithi110 ceph-mon[88262]: osdmap e513: 8 total, 7 up, 8 in 2024-09-14T03:45:16.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:15 smithi110 ceph-mon[86423]: osdmap e513: 8 total, 7 up, 8 in 2024-09-14T03:45:16.761 INFO:tasks.rados.rados.0.smithi110.stdout:2204: done (1 left) 2024-09-14T03:45:16.761 INFO:tasks.rados.rados.0.smithi110.stdout:2205: done (0 left) 2024-09-14T03:45:16.762 INFO:tasks.rados.rados.0.smithi110.stdout:2206: snap_remove snap 213 2024-09-14T03:45:17.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:16 smithi195 ceph-mon[82388]: osdmap e514: 8 total, 7 up, 8 in 2024-09-14T03:45:17.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:16 smithi195 ceph-mon[82388]: pgmap v400: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 263 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 13 op/s; 51/456 objects degraded (11.184%) 2024-09-14T03:45:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:16 smithi110 ceph-mon[88262]: osdmap e514: 8 total, 7 up, 8 in 2024-09-14T03:45:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:16 smithi110 ceph-mon[88262]: pgmap v400: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 263 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 13 op/s; 51/456 objects degraded (11.184%) 2024-09-14T03:45:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:16 smithi110 ceph-mon[86423]: osdmap e514: 8 total, 7 up, 8 in 2024-09-14T03:45:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:16 smithi110 ceph-mon[86423]: pgmap v400: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 263 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 13 op/s; 51/456 objects degraded (11.184%) 2024-09-14T03:45:17.764 INFO:tasks.rados.rados.0.smithi110.stdout:2206: done (0 left) 2024-09-14T03:45:17.764 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 224 2024-09-14T03:45:17.765 INFO:tasks.rados.rados.0.smithi110.stdout:2207: seq_num 701 ranges {2932736=548864,3481600=696320,4177920=540672,4718592=368640} 2024-09-14T03:45:17.784 INFO:tasks.rados.rados.0.smithi110.stdout:2207: writing smithi11075562-33 from 2932736 to 3481600 tid 1 2024-09-14T03:45:17.788 INFO:tasks.rados.rados.0.smithi110.stdout:2207: writing smithi11075562-33 from 3481600 to 4177920 tid 2 2024-09-14T03:45:17.790 INFO:tasks.rados.rados.0.smithi110.stdout:2207: writing smithi11075562-33 from 4177920 to 4718592 tid 3 2024-09-14T03:45:17.793 INFO:tasks.rados.rados.0.smithi110.stdout:2207: writing smithi11075562-33 from 4718592 to 5087232 tid 4 2024-09-14T03:45:17.793 INFO:tasks.rados.rados.0.smithi110.stdout:2208: snap_remove snap 196 2024-09-14T03:45:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:17 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 51/456 objects degraded (11.184%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:17 smithi110 ceph-mon[88262]: osdmap e515: 8 total, 7 up, 8 in 2024-09-14T03:45:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:17 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:17 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:18.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:17 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 51/456 objects degraded (11.184%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:18.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:17 smithi110 ceph-mon[86423]: osdmap e515: 8 total, 7 up, 8 in 2024-09-14T03:45:18.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:17 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:18.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:17 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:17 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 51/456 objects degraded (11.184%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:17 smithi195 ceph-mon[82388]: osdmap e515: 8 total, 7 up, 8 in 2024-09-14T03:45:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:17 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:17 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:18.708 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:18 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:45:18.424+0000 7f7695741640 -1 osd.6 505 log_to_monitors true 2024-09-14T03:45:18.784 INFO:tasks.rados.rados.0.smithi110.stdout:2208: done (1 left) 2024-09-14T03:45:18.784 INFO:tasks.rados.rados.0.smithi110.stdout:2209: read oid 16 snap -1 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2209: expect (ObjNum 613 snap 188 seq_num 613) 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2210: copy_from oid 22 from oid 49 current snap is 224 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2211: copy_from oid 43 from oid 50 current snap is 224 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2212: read oid 29 snap 223 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2212: expect (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2213: snap_create 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2207: finishing write tid 1 to smithi11075562-33 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2207: finishing write tid 2 to smithi11075562-33 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2207: finishing write tid 3 to smithi11075562-33 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2207: finishing write tid 4 to smithi11075562-33 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2207: finishing write tid 5 to smithi11075562-33 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:2207: finishing write tid 6 to smithi11075562-33 2024-09-14T03:45:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1465 (ObjNum 701 snap 224 seq_num 701) dirty exists 2024-09-14T03:45:18.786 INFO:tasks.rados.rados.0.smithi110.stdout:2207: left oid 33 (ObjNum 701 snap 224 seq_num 701) 2024-09-14T03:45:18.803 INFO:tasks.rados.rados.0.smithi110.stdout:2212: expect (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:45:18.849 INFO:tasks.rados.rados.0.smithi110.stdout:2210: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:45:18.849 INFO:tasks.rados.rados.0.smithi110.stdout:2209: expect (ObjNum 613 snap 188 seq_num 613) 2024-09-14T03:45:19.070 INFO:tasks.rados.rados.0.smithi110.stdout:2210: finishing copy_from to smithi11075562-22 2024-09-14T03:45:19.070 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1256 (ObjNum 660 snap 204 seq_num 660) dirty exists 2024-09-14T03:45:19.070 INFO:tasks.rados.rados.0.smithi110.stdout:2211: finishing copy_from racing read to smithi11075562-43 2024-09-14T03:45:19.071 INFO:tasks.rados.rados.0.smithi110.stdout:2211: finishing copy_from to smithi11075562-43 2024-09-14T03:45:19.071 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 952 (ObjNum 696 snap 219 seq_num 696) dirty exists 2024-09-14T03:45:19.080 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:45:18 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:45:18.805+0000 7f2bc0a21640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 113 after trimming was started 2024-09-14T03:45:19.080 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:18 smithi195 ceph-mon[82388]: osdmap e516: 8 total, 7 up, 8 in 2024-09-14T03:45:19.080 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:18 smithi195 ceph-mon[82388]: pgmap v403: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 263 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 10 op/s; 51/456 objects degraded (11.184%) 2024-09-14T03:45:19.080 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:18 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:19.081 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:18 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:19.081 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:18 smithi195 ceph-mon[82388]: from='osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:45:19.081 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:18 smithi195 ceph-mon[82388]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:45:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[88262]: osdmap e516: 8 total, 7 up, 8 in 2024-09-14T03:45:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[88262]: pgmap v403: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 263 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 10 op/s; 51/456 objects degraded (11.184%) 2024-09-14T03:45:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[88262]: from='osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:45:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[88262]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:45:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[86423]: osdmap e516: 8 total, 7 up, 8 in 2024-09-14T03:45:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[86423]: pgmap v403: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 263 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 10 op/s; 51/456 objects degraded (11.184%) 2024-09-14T03:45:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[86423]: from='osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:45:19.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:18 smithi110 ceph-mon[86423]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:45:19.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:45:19 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:45:19.078+0000 7f76894e3640 -1 osd.6 505 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:45:19.786 INFO:tasks.rados.rados.0.smithi110.stdout:2207: done (5 left) 2024-09-14T03:45:19.786 INFO:tasks.rados.rados.0.smithi110.stdout:2209: done (4 left) 2024-09-14T03:45:19.786 INFO:tasks.rados.rados.0.smithi110.stdout:2210: done (3 left) 2024-09-14T03:45:19.786 INFO:tasks.rados.rados.0.smithi110.stdout:2211: done (2 left) 2024-09-14T03:45:19.786 INFO:tasks.rados.rados.0.smithi110.stdout:2212: done (1 left) 2024-09-14T03:45:19.786 INFO:tasks.rados.rados.0.smithi110.stdout:2213: done (0 left) 2024-09-14T03:45:19.786 INFO:tasks.rados.rados.0.smithi110.stdout:2214: snap_remove snap 218 2024-09-14T03:45:19.903 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:19 smithi195 ceph-mon[82388]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:45:19.903 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:19 smithi195 ceph-mon[82388]: osdmap e517: 8 total, 7 up, 8 in 2024-09-14T03:45:19.903 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:19 smithi195 ceph-mon[82388]: from='osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:19.903 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:19 smithi195 ceph-mon[82388]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:19.903 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:19 smithi195 ceph-mon[82388]: from='osd.6 ' entity='osd.6' 2024-09-14T03:45:20.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[88262]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:45:20.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[88262]: osdmap e517: 8 total, 7 up, 8 in 2024-09-14T03:45:20.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[88262]: from='osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:20.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[88262]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:20.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[88262]: from='osd.6 ' entity='osd.6' 2024-09-14T03:45:20.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[86423]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:45:20.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[86423]: osdmap e517: 8 total, 7 up, 8 in 2024-09-14T03:45:20.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[86423]: from='osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:20.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[86423]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:20.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:19 smithi110 ceph-mon[86423]: from='osd.6 ' entity='osd.6' 2024-09-14T03:45:20.640 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:45:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:45:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:45:20.789 INFO:tasks.rados.rados.0.smithi110.stdout:2214: done (0 left) 2024-09-14T03:45:20.790 INFO:tasks.rados.rados.0.smithi110.stdout:2215: delete oid 41 current snap is 225 2024-09-14T03:45:20.803 INFO:tasks.rados.rados.0.smithi110.stdout:2215: done (0 left) 2024-09-14T03:45:20.803 INFO:tasks.rados.rados.0.smithi110.stdout:2216: delete oid 21 current snap is 225 2024-09-14T03:45:20.808 INFO:tasks.rados.rados.0.smithi110.stdout:2216: done (0 left) 2024-09-14T03:45:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:2217: read oid 13 snap 206 2024-09-14T03:45:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:2217: expect (ObjNum 550 snap 167 seq_num 550) 2024-09-14T03:45:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:2218: copy_from oid 38 from oid 45 current snap is 225 2024-09-14T03:45:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 225 2024-09-14T03:45:20.809 INFO:tasks.rados.rados.0.smithi110.stdout:2219: seq_num 704 ranges {925696=598016,1523712=745472,2269184=712704,2981888=147456} 2024-09-14T03:45:20.817 INFO:tasks.rados.rados.0.smithi110.stdout:2219: writing smithi11075562-46 from 925696 to 1523712 tid 1 2024-09-14T03:45:20.822 INFO:tasks.rados.rados.0.smithi110.stdout:2219: writing smithi11075562-46 from 1523712 to 2269184 tid 2 2024-09-14T03:45:20.827 INFO:tasks.rados.rados.0.smithi110.stdout:2219: writing smithi11075562-46 from 2269184 to 2981888 tid 3 2024-09-14T03:45:20.828 INFO:tasks.rados.rados.0.smithi110.stdout:2219: writing smithi11075562-46 from 2981888 to 3129344 tid 4 2024-09-14T03:45:20.828 INFO:tasks.rados.rados.0.smithi110.stdout:2220: delete oid 19 current snap is 225 2024-09-14T03:45:20.828 INFO:tasks.rados.rados.0.smithi110.stdout:2219: finishing write tid 1 to smithi11075562-46 2024-09-14T03:45:20.828 INFO:tasks.rados.rados.0.smithi110.stdout:2218: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:45:20.828 INFO:tasks.rados.rados.0.smithi110.stdout:2218: finishing copy_from to smithi11075562-38 2024-09-14T03:45:20.828 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1562 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:45:20.829 INFO:tasks.rados.rados.0.smithi110.stdout:2219: finishing write tid 2 to smithi11075562-46 2024-09-14T03:45:20.834 INFO:tasks.rados.rados.0.smithi110.stdout:2219: finishing write tid 3 to smithi11075562-46 2024-09-14T03:45:20.835 INFO:tasks.rados.rados.0.smithi110.stdout:2219: finishing write tid 4 to smithi11075562-46 2024-09-14T03:45:20.835 INFO:tasks.rados.rados.0.smithi110.stdout:2217: expect (ObjNum 550 snap 167 seq_num 550) 2024-09-14T03:45:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:20 smithi195 ceph-mon[82388]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:45:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:20 smithi195 ceph-mon[82388]: pgmap v405: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 232 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 569 KiB/s wr, 2 op/s; 52/423 objects degraded (12.293%) 2024-09-14T03:45:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:20 smithi195 ceph-mon[82388]: osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419] boot 2024-09-14T03:45:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:20 smithi195 ceph-mon[82388]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T03:45:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:20 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:45:21.120 INFO:tasks.rados.rados.0.smithi110.stdout:2219: finishing write tid 5 to smithi11075562-46 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2219: finishing write tid 6 to smithi11075562-46 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1470 (ObjNum 704 snap 225 seq_num 704) dirty exists 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2219: left oid 46 (ObjNum 704 snap 225 seq_num 704) 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2217: done (3 left) 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2218: done (2 left) 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2219: done (1 left) 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2220: done (0 left) 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2221: rmattr oid 41 current snap is 225 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2221: done (0 left) 2024-09-14T03:45:21.121 INFO:tasks.rados.rados.0.smithi110.stdout:2222: snap_create 2024-09-14T03:45:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[88262]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:45:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[88262]: pgmap v405: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 232 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 569 KiB/s wr, 2 op/s; 52/423 objects degraded (12.293%) 2024-09-14T03:45:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[88262]: osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419] boot 2024-09-14T03:45:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[88262]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T03:45:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:45:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[86423]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:45:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[86423]: pgmap v405: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 232 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 569 KiB/s wr, 2 op/s; 52/423 objects degraded (12.293%) 2024-09-14T03:45:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[86423]: osd.6 [v2:172.21.15.195:6816/977836419,v1:172.21.15.195:6817/977836419] boot 2024-09-14T03:45:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[86423]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T03:45:21.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:20 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:45:21.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:45:21.430+0000 7f940e88b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-14T03:45:21.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:45:21] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.51.0" 2024-09-14T03:45:21.790 INFO:tasks.rados.rados.0.smithi110.stdout:2222: done (0 left) 2024-09-14T03:45:21.790 INFO:tasks.rados.rados.0.smithi110.stdout:2223: rollback oid 36 current snap is 226 2024-09-14T03:45:21.790 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 36 to 219 2024-09-14T03:45:21.790 INFO:tasks.rados.rados.0.smithi110.stdout:2224: delete oid 18 current snap is 226 2024-09-14T03:45:21.793 INFO:tasks.rados.rados.0.smithi110.stdout:2224: done (1 left) 2024-09-14T03:45:21.793 INFO:tasks.rados.rados.0.smithi110.stdout:2225: read oid 49 snap -1 2024-09-14T03:45:21.793 INFO:tasks.rados.rados.0.smithi110.stdout:2225: expect (ObjNum 660 snap 204 seq_num 660) 2024-09-14T03:45:21.793 INFO:tasks.rados.rados.0.smithi110.stdout:2226: copy_from oid 32 from oid 1 current snap is 226 2024-09-14T03:45:21.793 INFO:tasks.rados.rados.0.smithi110.stdout:2227: rollback oid 43 current snap is 226 2024-09-14T03:45:21.794 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 43 to 206 2024-09-14T03:45:21.794 INFO:tasks.rados.rados.0.smithi110.stdout:2228: rollback oid 41 current snap is 226 2024-09-14T03:45:21.794 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 219 2024-09-14T03:45:21.794 INFO:tasks.rados.rados.0.smithi110.stdout:2229: snap_create 2024-09-14T03:45:21.797 INFO:tasks.rados.rados.0.smithi110.stdout:2223: finishing rollback tid 1 to smithi11075562-36 2024-09-14T03:45:21.797 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1481 (ObjNum 129 snap 0 seq_num 66509952) dirty dne 2024-09-14T03:45:21.799 INFO:tasks.rados.rados.0.smithi110.stdout:2227: finishing rollback tid 0 to smithi11075562-43 2024-09-14T03:45:21.801 INFO:tasks.rados.rados.0.smithi110.stdout:2227: finishing rollback tid 1 to smithi11075562-43 2024-09-14T03:45:21.801 INFO:tasks.rados.rados.0.smithi110.stdout:2227: finishing rollback tid 2 to smithi11075562-43 2024-09-14T03:45:21.801 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1486 (ObjNum 613 snap 188 seq_num 613) dirty exists 2024-09-14T03:45:21.801 INFO:tasks.rados.rados.0.smithi110.stdout:2228: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:45:21.801 INFO:tasks.rados.rados.0.smithi110.stdout:2228: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:45:21.802 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1488 (ObjNum 33 snap 0 seq_num 1495384433) dirty exists 2024-09-14T03:45:21.805 INFO:tasks.rados.rados.0.smithi110.stdout:2225: expect (ObjNum 660 snap 204 seq_num 660) 2024-09-14T03:45:21.879 INFO:tasks.rados.rados.0.smithi110.stdout:2226: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:45:21.879 INFO:tasks.rados.rados.0.smithi110.stdout:2226: finishing copy_from to smithi11075562-32 2024-09-14T03:45:21.879 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1276 (ObjNum 572 snap 174 seq_num 572) dirty exists 2024-09-14T03:45:22.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T03:45:22.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:22.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:45:22.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:22.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:45:22.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:22.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:21 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:21 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:45:22.797 INFO:tasks.rados.rados.0.smithi110.stdout:2223: done (5 left) 2024-09-14T03:45:22.797 INFO:tasks.rados.rados.0.smithi110.stdout:2225: done (4 left) 2024-09-14T03:45:22.797 INFO:tasks.rados.rados.0.smithi110.stdout:2226: done (3 left) 2024-09-14T03:45:22.797 INFO:tasks.rados.rados.0.smithi110.stdout:2227: done (2 left) 2024-09-14T03:45:22.797 INFO:tasks.rados.rados.0.smithi110.stdout:2228: done (1 left) 2024-09-14T03:45:22.797 INFO:tasks.rados.rados.0.smithi110.stdout:2229: done (0 left) 2024-09-14T03:45:22.797 INFO:tasks.rados.rados.0.smithi110.stdout:2230: read oid 42 snap 206 2024-09-14T03:45:22.798 INFO:tasks.rados.rados.0.smithi110.stdout:2230: expect (ObjNum 643 snap 199 seq_num 643) 2024-09-14T03:45:22.798 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 227 2024-09-14T03:45:22.798 INFO:tasks.rados.rados.0.smithi110.stdout:2231: seq_num 707 ranges {0=647168,647168=802816,1449984=532480,1982464=270336} 2024-09-14T03:45:22.800 INFO:tasks.rados.rados.0.smithi110.stdout:2231: writing smithi11075562-12 from 0 to 647168 tid 1 2024-09-14T03:45:22.805 INFO:tasks.rados.rados.0.smithi110.stdout:2231: writing smithi11075562-12 from 647168 to 1449984 tid 2 2024-09-14T03:45:22.808 INFO:tasks.rados.rados.0.smithi110.stdout:2231: writing smithi11075562-12 from 1449984 to 1982464 tid 3 2024-09-14T03:45:22.809 INFO:tasks.rados.rados.0.smithi110.stdout:2231: writing smithi11075562-12 from 1982464 to 2252800 tid 4 2024-09-14T03:45:22.809 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 227 2024-09-14T03:45:22.810 INFO:tasks.rados.rados.0.smithi110.stdout:2232: seq_num 708 ranges {2342912=663552,3006464=737280,3743744=696320,4440064=172032} 2024-09-14T03:45:22.825 INFO:tasks.rados.rados.0.smithi110.stdout:2232: writing smithi11075562-22 from 2342912 to 3006464 tid 1 2024-09-14T03:45:22.829 INFO:tasks.rados.rados.0.smithi110.stdout:2232: writing smithi11075562-22 from 3006464 to 3743744 tid 2 2024-09-14T03:45:22.833 INFO:tasks.rados.rados.0.smithi110.stdout:2232: writing smithi11075562-22 from 3743744 to 4440064 tid 3 2024-09-14T03:45:22.835 INFO:tasks.rados.rados.0.smithi110.stdout:2232: writing smithi11075562-22 from 4440064 to 4612096 tid 4 2024-09-14T03:45:22.835 INFO:tasks.rados.rados.0.smithi110.stdout:2233: rollback oid 35 current snap is 227 2024-09-14T03:45:22.835 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 214 2024-09-14T03:45:22.835 INFO:tasks.rados.rados.0.smithi110.stdout:2231: finishing write tid 1 to smithi11075562-12 2024-09-14T03:45:22.835 INFO:tasks.rados.rados.0.smithi110.stdout:2234: rollback oid 7 current snap is 227 2024-09-14T03:45:22.835 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 7 to 216 2024-09-14T03:45:22.836 INFO:tasks.rados.rados.0.smithi110.stdout:2230: expect (ObjNum 643 snap 199 seq_num 643) 2024-09-14T03:45:22.903 INFO:tasks.rados.rados.0.smithi110.stdout:2231: finishing write tid 2 to smithi11075562-12 2024-09-14T03:45:22.903 INFO:tasks.rados.rados.0.smithi110.stdout:2231: finishing write tid 3 to smithi11075562-12 2024-09-14T03:45:22.903 INFO:tasks.rados.rados.0.smithi110.stdout:2231: finishing write tid 4 to smithi11075562-12 2024-09-14T03:45:22.903 INFO:tasks.rados.rados.0.smithi110.stdout:2231: finishing write tid 5 to smithi11075562-12 2024-09-14T03:45:22.903 INFO:tasks.rados.rados.0.smithi110.stdout:2231: finishing write tid 6 to smithi11075562-12 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1594 (ObjNum 707 snap 227 seq_num 707) dirty exists 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2231: left oid 12 (ObjNum 707 snap 227 seq_num 707) 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2232: finishing write tid 1 to smithi11075562-22 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2230: done (4 left) 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2231: done (3 left) 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2235: read oid 15 snap -1 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2235: expect deleted 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2236: rmattr oid 2 current snap is 227 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2236: done (4 left) 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2237: setattr oid 48 current snap is 227 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2232: finishing write tid 2 to smithi11075562-22 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2233: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 0 (ObjNum 33 snap 0 seq_num 43181432) dirty dne 2024-09-14T03:45:22.904 INFO:tasks.rados.rados.0.smithi110.stdout:2232: finishing write tid 3 to smithi11075562-22 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2232: finishing write tid 4 to smithi11075562-22 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2232: finishing write tid 5 to smithi11075562-22 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2234: finishing rollback tid 0 to smithi11075562-7 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2234: finishing rollback tid 1 to smithi11075562-7 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2232: finishing write tid 6 to smithi11075562-22 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1346 (ObjNum 708 snap 227 seq_num 708) dirty exists 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2232: left oid 22 (ObjNum 708 snap 227 seq_num 708) 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2234: finishing rollback tid 2 to smithi11075562-7 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1565 (ObjNum 691 snap 216 seq_num 691) dirty exists 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2232: done (4 left) 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2233: done (3 left) 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2234: done (2 left) 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2235: done (1 left) 2024-09-14T03:45:22.905 INFO:tasks.rados.rados.0.smithi110.stdout:2238: delete oid 17 current snap is 227 2024-09-14T03:45:22.906 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1123 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:45:22.907 INFO:tasks.rados.rados.0.smithi110.stdout:2237: done (1 left) 2024-09-14T03:45:22.907 INFO:tasks.rados.rados.0.smithi110.stdout:2238: done (0 left) 2024-09-14T03:45:22.907 INFO:tasks.rados.rados.0.smithi110.stdout:2239: copy_from oid 23 from oid 37 current snap is 227 2024-09-14T03:45:22.908 INFO:tasks.rados.rados.0.smithi110.stdout:2240: delete oid 17 current snap is 227 2024-09-14T03:45:22.909 INFO:tasks.rados.rados.0.smithi110.stdout:2240: done (1 left) 2024-09-14T03:45:22.909 INFO:tasks.rados.rados.0.smithi110.stdout:2241: read oid 29 snap -1 2024-09-14T03:45:22.909 INFO:tasks.rados.rados.0.smithi110.stdout:2241: expect (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:45:22.909 INFO:tasks.rados.rados.0.smithi110.stdout:2242: delete oid 35 current snap is 227 2024-09-14T03:45:22.911 INFO:tasks.rados.rados.0.smithi110.stdout:2242: done (2 left) 2024-09-14T03:45:22.911 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 227 2024-09-14T03:45:22.911 INFO:tasks.rados.rados.0.smithi110.stdout:2243: seq_num 712 ranges {0=729088,729088=671744,1400832=671744,2072576=262144} 2024-09-14T03:45:22.915 INFO:tasks.rados.rados.0.smithi110.stdout:2243: writing smithi11075562-44 from 0 to 729088 tid 1 2024-09-14T03:45:22.919 INFO:tasks.rados.rados.0.smithi110.stdout:2243: writing smithi11075562-44 from 729088 to 1400832 tid 2 2024-09-14T03:45:22.923 INFO:tasks.rados.rados.0.smithi110.stdout:2243: writing smithi11075562-44 from 1400832 to 2072576 tid 3 2024-09-14T03:45:22.925 INFO:tasks.rados.rados.0.smithi110.stdout:2243: writing smithi11075562-44 from 2072576 to 2334720 tid 4 2024-09-14T03:45:22.925 INFO:tasks.rados.rados.0.smithi110.stdout:2244: rollback oid 17 current snap is 227 2024-09-14T03:45:22.925 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 223 2024-09-14T03:45:22.925 INFO:tasks.rados.rados.0.smithi110.stdout:2241: expect (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2243: finishing write tid 1 to smithi11075562-44 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2243: finishing write tid 2 to smithi11075562-44 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2241: done (3 left) 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2245: delete oid 16 current snap is 227 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2243: finishing write tid 3 to smithi11075562-44 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2243: finishing write tid 4 to smithi11075562-44 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2244: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2243: finishing write tid 5 to smithi11075562-44 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2239: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:2243: finishing write tid 6 to smithi11075562-44 2024-09-14T03:45:22.964 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 407 (ObjNum 712 snap 227 seq_num 712) dirty exists 2024-09-14T03:45:22.965 INFO:tasks.rados.rados.0.smithi110.stdout:2243: left oid 44 (ObjNum 712 snap 227 seq_num 712) 2024-09-14T03:45:22.965 INFO:tasks.rados.rados.0.smithi110.stdout:2244: finishing rollback tid 2 to smithi11075562-17 2024-09-14T03:45:22.965 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1126 (ObjNum 668 snap 206 seq_num 668) dirty exists 2024-09-14T03:45:22.965 INFO:tasks.rados.rados.0.smithi110.stdout:2239: finishing copy_from to smithi11075562-23 2024-09-14T03:45:22.965 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1479 (ObjNum 592 snap 181 seq_num 592) dirty exists 2024-09-14T03:45:22.966 INFO:tasks.rados.rados.0.smithi110.stdout:2239: done (3 left) 2024-09-14T03:45:22.966 INFO:tasks.rados.rados.0.smithi110.stdout:2243: done (2 left) 2024-09-14T03:45:22.966 INFO:tasks.rados.rados.0.smithi110.stdout:2244: done (1 left) 2024-09-14T03:45:22.966 INFO:tasks.rados.rados.0.smithi110.stdout:2245: done (0 left) 2024-09-14T03:45:22.966 INFO:tasks.rados.rados.0.smithi110.stdout:2246: setattr oid 29 current snap is 227 2024-09-14T03:45:22.967 INFO:tasks.rados.rados.0.smithi110.stdout:2247: rollback oid 3 current snap is 227 2024-09-14T03:45:22.967 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 3 to 223 2024-09-14T03:45:22.967 INFO:tasks.rados.rados.0.smithi110.stdout:2248: rollback oid 20 current snap is 227 2024-09-14T03:45:22.967 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 220 2024-09-14T03:45:22.967 INFO:tasks.rados.rados.0.smithi110.stdout:2249: read oid 43 snap 214 2024-09-14T03:45:22.967 INFO:tasks.rados.rados.0.smithi110.stdout:2249: expect (ObjNum 679 snap 212 seq_num 679) 2024-09-14T03:45:22.968 INFO:tasks.rados.rados.0.smithi110.stdout:2250: delete oid 8 current snap is 227 2024-09-14T03:45:22.970 INFO:tasks.rados.rados.0.smithi110.stdout:2247: finishing rollback tid 1 to smithi11075562-3 2024-09-14T03:45:22.970 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 785 (ObjNum 33 snap 0 seq_num 7) dirty dne 2024-09-14T03:45:22.971 INFO:tasks.rados.rados.0.smithi110.stdout:2247: done (4 left) 2024-09-14T03:45:22.971 INFO:tasks.rados.rados.0.smithi110.stdout:2250: done (3 left) 2024-09-14T03:45:22.971 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 227 2024-09-14T03:45:22.971 INFO:tasks.rados.rados.0.smithi110.stdout:2251: seq_num 715 ranges {524288=778240,1302528=770048,2072576=499712,2572288=335872} 2024-09-14T03:45:22.978 INFO:tasks.rados.rados.0.smithi110.stdout:2251: writing smithi11075562-11 from 524288 to 1302528 tid 1 2024-09-14T03:45:22.983 INFO:tasks.rados.rados.0.smithi110.stdout:2251: writing smithi11075562-11 from 1302528 to 2072576 tid 2 2024-09-14T03:45:22.986 INFO:tasks.rados.rados.0.smithi110.stdout:2251: writing smithi11075562-11 from 2072576 to 2572288 tid 3 2024-09-14T03:45:22.988 INFO:tasks.rados.rados.0.smithi110.stdout:2251: writing smithi11075562-11 from 2572288 to 2908160 tid 4 2024-09-14T03:45:22.988 INFO:tasks.rados.rados.0.smithi110.stdout:2252: delete oid 48 current snap is 227 2024-09-14T03:45:22.988 INFO:tasks.rados.rados.0.smithi110.stdout:2248: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:45:22.988 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1566 (ObjNum 653 snap 202 seq_num 653) dirty exists 2024-09-14T03:45:22.988 INFO:tasks.rados.rados.0.smithi110.stdout:2248: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:45:22.988 INFO:tasks.rados.rados.0.smithi110.stdout:2248: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:45:22.989 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1473 (ObjNum 663 snap 204 seq_num 663) dirty exists 2024-09-14T03:45:22.989 INFO:tasks.rados.rados.0.smithi110.stdout:2251: finishing write tid 1 to smithi11075562-11 2024-09-14T03:45:22.989 INFO:tasks.rados.rados.0.smithi110.stdout:2251: finishing write tid 2 to smithi11075562-11 2024-09-14T03:45:22.990 INFO:tasks.rados.rados.0.smithi110.stdout:2251: finishing write tid 3 to smithi11075562-11 2024-09-14T03:45:22.992 INFO:tasks.rados.rados.0.smithi110.stdout:2251: finishing write tid 4 to smithi11075562-11 2024-09-14T03:45:22.992 INFO:tasks.rados.rados.0.smithi110.stdout:2251: finishing write tid 5 to smithi11075562-11 2024-09-14T03:45:22.992 INFO:tasks.rados.rados.0.smithi110.stdout:2249: expect (ObjNum 679 snap 212 seq_num 679) 2024-09-14T03:45:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:45:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[88262]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-14T03:45:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[88262]: pgmap v408: 121 pgs: 6 peering, 26 active+undersized, 8 active+undersized+degraded, 81 active+clean; 231 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 570 KiB/s wr, 4 op/s; 34/408 objects degraded (8.333%) 2024-09-14T03:45:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[88262]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T03:45:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 34/408 objects degraded (8.333%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:45:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[86423]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-14T03:45:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[86423]: pgmap v408: 121 pgs: 6 peering, 26 active+undersized, 8 active+undersized+degraded, 81 active+clean; 231 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 570 KiB/s wr, 4 op/s; 34/408 objects degraded (8.333%) 2024-09-14T03:45:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[86423]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T03:45:23.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:22 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 34/408 objects degraded (8.333%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:23.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:22 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:45:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:22 smithi195 ceph-mon[82388]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-14T03:45:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:22 smithi195 ceph-mon[82388]: pgmap v408: 121 pgs: 6 peering, 26 active+undersized, 8 active+undersized+degraded, 81 active+clean; 231 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 570 KiB/s wr, 4 op/s; 34/408 objects degraded (8.333%) 2024-09-14T03:45:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:22 smithi195 ceph-mon[82388]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T03:45:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:22 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 34/408 objects degraded (8.333%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:23.332 INFO:tasks.rados.rados.0.smithi110.stdout:2251: finishing write tid 6 to smithi11075562-11 2024-09-14T03:45:23.332 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1131 (ObjNum 715 snap 227 seq_num 715) dirty exists 2024-09-14T03:45:23.333 INFO:tasks.rados.rados.0.smithi110.stdout:2251: left oid 11 (ObjNum 715 snap 227 seq_num 715) 2024-09-14T03:45:23.333 INFO:tasks.rados.rados.0.smithi110.stdout:2246: done (4 left) 2024-09-14T03:45:23.333 INFO:tasks.rados.rados.0.smithi110.stdout:2248: done (3 left) 2024-09-14T03:45:23.333 INFO:tasks.rados.rados.0.smithi110.stdout:2249: done (2 left) 2024-09-14T03:45:23.333 INFO:tasks.rados.rados.0.smithi110.stdout:2251: done (1 left) 2024-09-14T03:45:23.333 INFO:tasks.rados.rados.0.smithi110.stdout:2252: done (0 left) 2024-09-14T03:45:23.333 INFO:tasks.rados.rados.0.smithi110.stdout:2253: snap_remove snap 220 2024-09-14T03:45:23.806 INFO:tasks.rados.rados.0.smithi110.stdout:2253: done (0 left) 2024-09-14T03:45:23.806 INFO:tasks.rados.rados.0.smithi110.stdout:2254: snap_remove snap 224 2024-09-14T03:45:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:23 smithi110 ceph-mon[88262]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T03:45:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:23 smithi110 ceph-mon[88262]: pgmap v411: 121 pgs: 6 peering, 14 active+undersized, 4 active+undersized+degraded, 97 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.0 MiB/s wr, 8 op/s; 20/393 objects degraded (5.089%); 1.9 MiB/s, 0 objects/s recovering 2024-09-14T03:45:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:23 smithi110 ceph-mon[86423]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T03:45:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:23 smithi110 ceph-mon[86423]: pgmap v411: 121 pgs: 6 peering, 14 active+undersized, 4 active+undersized+degraded, 97 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.0 MiB/s wr, 8 op/s; 20/393 objects degraded (5.089%); 1.9 MiB/s, 0 objects/s recovering 2024-09-14T03:45:24.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:23 smithi195 ceph-mon[82388]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T03:45:24.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:23 smithi195 ceph-mon[82388]: pgmap v411: 121 pgs: 6 peering, 14 active+undersized, 4 active+undersized+degraded, 97 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.0 MiB/s wr, 8 op/s; 20/393 objects degraded (5.089%); 1.9 MiB/s, 0 objects/s recovering 2024-09-14T03:45:24.809 INFO:tasks.rados.rados.0.smithi110.stdout:2254: done (0 left) 2024-09-14T03:45:24.809 INFO:tasks.rados.rados.0.smithi110.stdout:2255: rollback oid 34 current snap is 227 2024-09-14T03:45:24.809 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 225 2024-09-14T03:45:24.810 INFO:tasks.rados.rados.0.smithi110.stdout:2256: snap_remove snap 223 2024-09-14T03:45:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:24 smithi110 ceph-mon[88262]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T03:45:25.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:24 smithi110 ceph-mon[86423]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T03:45:25.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:24 smithi195 ceph-mon[82388]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T03:45:25.810 INFO:tasks.rados.rados.0.smithi110.stdout:2256: done (1 left) 2024-09-14T03:45:25.810 INFO:tasks.rados.rados.0.smithi110.stdout:2257: copy_from oid 27 from oid 37 current snap is 227 2024-09-14T03:45:25.810 INFO:tasks.rados.rados.0.smithi110.stdout:2255: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:45:25.810 INFO:tasks.rados.rados.0.smithi110.stdout:2255: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:45:25.810 INFO:tasks.rados.rados.0.smithi110.stdout:2255: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:45:25.810 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1207 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:45:25.811 INFO:tasks.rados.rados.0.smithi110.stdout:2255: done (1 left) 2024-09-14T03:45:25.811 INFO:tasks.rados.rados.0.smithi110.stdout:2258: snap_remove snap 219 2024-09-14T03:45:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:25 smithi110 ceph-mon[88262]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T03:45:26.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:25 smithi110 ceph-mon[88262]: pgmap v414: 121 pgs: 121 active+clean; 246 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1002 KiB/s rd, 6.3 MiB/s wr, 17 op/s; 6.9 MiB/s, 2 objects/s recovering 2024-09-14T03:45:26.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:25 smithi110 ceph-mon[88262]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T03:45:26.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:25 smithi110 ceph-mon[86423]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T03:45:26.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:25 smithi110 ceph-mon[86423]: pgmap v414: 121 pgs: 121 active+clean; 246 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1002 KiB/s rd, 6.3 MiB/s wr, 17 op/s; 6.9 MiB/s, 2 objects/s recovering 2024-09-14T03:45:26.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:25 smithi110 ceph-mon[86423]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T03:45:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:25 smithi195 ceph-mon[82388]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T03:45:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:25 smithi195 ceph-mon[82388]: pgmap v414: 121 pgs: 121 active+clean; 246 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1002 KiB/s rd, 6.3 MiB/s wr, 17 op/s; 6.9 MiB/s, 2 objects/s recovering 2024-09-14T03:45:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:25 smithi195 ceph-mon[82388]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T03:45:26.812 INFO:tasks.rados.rados.0.smithi110.stdout:2258: done (1 left) 2024-09-14T03:45:26.812 INFO:tasks.rados.rados.0.smithi110.stdout:2259: delete oid 11 current snap is 227 2024-09-14T03:45:26.813 INFO:tasks.rados.rados.0.smithi110.stdout:2257: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:45:26.813 INFO:tasks.rados.rados.0.smithi110.stdout:2257: finishing copy_from to smithi11075562-27 2024-09-14T03:45:26.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1479 (ObjNum 592 snap 181 seq_num 592) dirty exists 2024-09-14T03:45:26.822 INFO:tasks.rados.rados.0.smithi110.stdout:2257: done (1 left) 2024-09-14T03:45:26.822 INFO:tasks.rados.rados.0.smithi110.stdout:2259: done (0 left) 2024-09-14T03:45:26.822 INFO:tasks.rados.rados.0.smithi110.stdout:2260: read oid 47 snap -1 2024-09-14T03:45:26.822 INFO:tasks.rados.rados.0.smithi110.stdout:2260: expect (ObjNum 674 snap 209 seq_num 674) 2024-09-14T03:45:26.822 INFO:tasks.rados.rados.0.smithi110.stdout:2261: rmattr oid 42 current snap is 227 2024-09-14T03:45:26.823 INFO:tasks.rados.rados.0.smithi110.stdout:2261: done (1 left) 2024-09-14T03:45:26.823 INFO:tasks.rados.rados.0.smithi110.stdout:2262: rmattr oid 12 current snap is 227 2024-09-14T03:45:26.823 INFO:tasks.rados.rados.0.smithi110.stdout:2262: done (1 left) 2024-09-14T03:45:26.823 INFO:tasks.rados.rados.0.smithi110.stdout:2263: rollback oid 27 current snap is 227 2024-09-14T03:45:26.823 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 216 2024-09-14T03:45:26.823 INFO:tasks.rados.rados.0.smithi110.stdout:2264: snap_create 2024-09-14T03:45:26.827 INFO:tasks.rados.rados.0.smithi110.stdout:2263: finishing rollback tid 0 to smithi11075562-27 2024-09-14T03:45:26.829 INFO:tasks.rados.rados.0.smithi110.stdout:2263: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:45:26.829 INFO:tasks.rados.rados.0.smithi110.stdout:2263: finishing rollback tid 2 to smithi11075562-27 2024-09-14T03:45:26.829 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1482 (ObjNum 623 snap 190 seq_num 623) dirty exists 2024-09-14T03:45:26.842 INFO:tasks.rados.rados.0.smithi110.stdout:2260: expect (ObjNum 674 snap 209 seq_num 674) 2024-09-14T03:45:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:26 smithi110 ceph-mon[88262]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/393 objects degraded (5.089%), 4 pgs degraded) 2024-09-14T03:45:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:26 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:45:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:26 smithi110 ceph-mon[88262]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T03:45:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:26 smithi110 ceph-mon[86423]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/393 objects degraded (5.089%), 4 pgs degraded) 2024-09-14T03:45:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:26 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:45:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:26 smithi110 ceph-mon[86423]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T03:45:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:26 smithi195 ceph-mon[82388]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/393 objects degraded (5.089%), 4 pgs degraded) 2024-09-14T03:45:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:26 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:45:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:26 smithi195 ceph-mon[82388]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T03:45:27.824 INFO:tasks.rados.rados.0.smithi110.stdout:2260: done (2 left) 2024-09-14T03:45:27.824 INFO:tasks.rados.rados.0.smithi110.stdout:2263: done (1 left) 2024-09-14T03:45:27.825 INFO:tasks.rados.rados.0.smithi110.stdout:2264: done (0 left) 2024-09-14T03:45:27.825 INFO:tasks.rados.rados.0.smithi110.stdout:2265: rollback oid 31 current snap is 228 2024-09-14T03:45:27.825 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 227 2024-09-14T03:45:27.825 INFO:tasks.rados.rados.0.smithi110.stdout:2266: snap_remove snap 222 2024-09-14T03:45:28.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:28 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:28.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:28 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:28.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:28 smithi110 ceph-mon[86423]: pgmap v417: 121 pgs: 121 active+clean; 249 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.9 MiB/s wr, 24 op/s; 5.0 MiB/s, 2 objects/s recovering 2024-09-14T03:45:28.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:28 smithi110 ceph-mon[86423]: osdmap e526: 8 total, 8 up, 8 in 2024-09-14T03:45:28.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:28 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:28.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:28 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:28.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:28 smithi110 ceph-mon[88262]: pgmap v417: 121 pgs: 121 active+clean; 249 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.9 MiB/s wr, 24 op/s; 5.0 MiB/s, 2 objects/s recovering 2024-09-14T03:45:28.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:28 smithi110 ceph-mon[88262]: osdmap e526: 8 total, 8 up, 8 in 2024-09-14T03:45:28.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:28 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:28.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:28 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:28.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:28 smithi195 ceph-mon[82388]: pgmap v417: 121 pgs: 121 active+clean; 249 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.9 MiB/s wr, 24 op/s; 5.0 MiB/s, 2 objects/s recovering 2024-09-14T03:45:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:28 smithi195 ceph-mon[82388]: osdmap e526: 8 total, 8 up, 8 in 2024-09-14T03:45:28.734 INFO:tasks.rados.rados.0.smithi110.stdout:2266: done (1 left) 2024-09-14T03:45:28.734 INFO:tasks.rados.rados.0.smithi110.stdout:2267: rmattr oid 21 current snap is 228 2024-09-14T03:45:28.734 INFO:tasks.rados.rados.0.smithi110.stdout:2267: done (1 left) 2024-09-14T03:45:28.734 INFO:tasks.rados.rados.0.smithi110.stdout:2268: snap_remove snap 206 2024-09-14T03:45:29.738 INFO:tasks.rados.rados.0.smithi110.stdout:2268: done (1 left) 2024-09-14T03:45:29.738 INFO:tasks.rados.rados.0.smithi110.stdout:2269: rollback oid 12 current snap is 228 2024-09-14T03:45:29.738 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 204 2024-09-14T03:45:29.738 INFO:tasks.rados.rados.0.smithi110.stdout:2265: finishing rollback tid 0 to smithi11075562-31 2024-09-14T03:45:29.738 INFO:tasks.rados.rados.0.smithi110.stdout:2265: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:45:29.738 INFO:tasks.rados.rados.0.smithi110.stdout:2265: finishing rollback tid 2 to smithi11075562-31 2024-09-14T03:45:29.739 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1210 (ObjNum 623 snap 190 seq_num 623) dirty exists 2024-09-14T03:45:29.739 INFO:tasks.rados.rados.0.smithi110.stdout:2265: done (1 left) 2024-09-14T03:45:29.739 INFO:tasks.rados.rados.0.smithi110.stdout:2270: read oid 11 snap -1 2024-09-14T03:45:29.739 INFO:tasks.rados.rados.0.smithi110.stdout:2270: expect deleted 2024-09-14T03:45:29.739 INFO:tasks.rados.rados.0.smithi110.stdout:2271: snap_create 2024-09-14T03:45:29.750 INFO:tasks.rados.rados.0.smithi110.stdout:2269: finishing rollback tid 0 to smithi11075562-12 2024-09-14T03:45:29.756 INFO:tasks.rados.rados.0.smithi110.stdout:2269: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:45:29.756 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1599 (ObjNum 33 snap 0 seq_num 0) dirty dne 2024-09-14T03:45:30.037 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:45:30.038 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:29 smithi110 ceph-mon[88262]: osdmap e527: 8 total, 8 up, 8 in 2024-09-14T03:45:30.038 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:29 smithi110 ceph-mon[86423]: osdmap e527: 8 total, 8 up, 8 in 2024-09-14T03:45:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:29 smithi195 ceph-mon[82388]: osdmap e527: 8 total, 8 up, 8 in 2024-09-14T03:45:30.584 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (6m) 114s ago 10m 19.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (5m) 13s ago 10m 82.8M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (11m) 114s ago 11m 20.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (11m) 13s ago 11m 17.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (5m) 13s ago 15m 455M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (5m) 114s ago 16m 525M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (4m) 114s ago 16m 55.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (4m) 13s ago 15m 46.4M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (4m) 114s ago 15m 49.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (6m) 114s ago 9m 9.92M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:45:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (5m) 13s ago 9m 11.0M - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (3m) 114s ago 14m 218M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (3m) 114s ago 14m 117M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (2m) 114s ago 13m 101M 2480M 19.3.0-4907-ga2782315 8801d2529e43 0cdc58c38711 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (118s) 114s ago 13m 14.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2283ebcb5f10 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (85s) 13s ago 13m 177M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6921b0a30047 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (51s) 13s ago 12m 125M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8fe7f08d5882 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (17s) 13s ago 12m 11.5M 2480M 19.3.0-4907-ga2782315 8801d2529e43 5305ca81bc26 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (12m) 13s ago 12m 196M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e5b6950370c8 2024-09-14T03:45:30.586 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (5m) 13s ago 10m 54.8M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:45:30.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:45:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:45:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:45:30.747 INFO:tasks.rados.rados.0.smithi110.stdout:2269: done (2 left) 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2270: done (1 left) 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2271: done (0 left) 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2272: copy_from oid 21 from oid 50 current snap is 229 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2273: read oid 7 snap -1 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2273: expect (ObjNum 691 snap 216 seq_num 691) 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2274: read oid 37 snap -1 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2274: expect (ObjNum 592 snap 181 seq_num 592) 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2275: rmattr oid 29 current snap is 229 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 229 2024-09-14T03:45:30.748 INFO:tasks.rados.rados.0.smithi110.stdout:2276: seq_num 718 ranges {2203648=32768} 2024-09-14T03:45:30.760 INFO:tasks.rados.rados.0.smithi110.stdout:2276: writing smithi11075562-38 from 2203648 to 2236416 tid 1 2024-09-14T03:45:30.760 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 229 2024-09-14T03:45:30.761 INFO:tasks.rados.rados.0.smithi110.stdout:2277: seq_num 719 ranges {0=49152} 2024-09-14T03:45:30.761 INFO:tasks.rados.rados.0.smithi110.stdout:2277: writing smithi11075562-4 from 0 to 49152 tid 1 2024-09-14T03:45:30.761 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 229 2024-09-14T03:45:30.761 INFO:tasks.rados.rados.0.smithi110.stdout:2278: seq_num 720 ranges {4005888=65536} 2024-09-14T03:45:30.784 INFO:tasks.rados.rados.0.smithi110.stdout:2278: writing smithi11075562-42 from 4005888 to 4071424 tid 1 2024-09-14T03:45:30.784 INFO:tasks.rados.rados.0.smithi110.stdout:2279: read oid 12 snap -1 2024-09-14T03:45:30.784 INFO:tasks.rados.rados.0.smithi110.stdout:2279: expect deleted 2024-09-14T03:45:30.784 INFO:tasks.rados.rados.0.smithi110.stdout:2280: rollback oid 17 current snap is 229 2024-09-14T03:45:30.784 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 204 2024-09-14T03:45:30.784 INFO:tasks.rados.rados.0.smithi110.stdout:2273: expect (ObjNum 691 snap 216 seq_num 691) 2024-09-14T03:45:30.836 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1567 (ObjNum 653 snap 202 seq_num 653) dirty exists 2024-09-14T03:45:30.836 INFO:tasks.rados.rados.0.smithi110.stdout:2276: finishing write tid 1 to smithi11075562-38 2024-09-14T03:45:30.836 INFO:tasks.rados.rados.0.smithi110.stdout:2277: finishing write tid 1 to smithi11075562-4 2024-09-14T03:45:30.836 INFO:tasks.rados.rados.0.smithi110.stdout:2276: finishing write tid 2 to smithi11075562-38 2024-09-14T03:45:30.836 INFO:tasks.rados.rados.0.smithi110.stdout:2273: done (8 left) 2024-09-14T03:45:30.836 INFO:tasks.rados.rados.0.smithi110.stdout:2275: done (7 left) 2024-09-14T03:45:30.836 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 229 2024-09-14T03:45:30.837 INFO:tasks.rados.rados.0.smithi110.stdout:2277: finishing write tid 2 to smithi11075562-4 2024-09-14T03:45:30.837 INFO:tasks.rados.rados.0.smithi110.stdout:2277: finishing write tid 3 to smithi11075562-4 2024-09-14T03:45:30.837 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1212 (ObjNum 719 snap 229 seq_num 719) dirty exists 2024-09-14T03:45:30.837 INFO:tasks.rados.rados.0.smithi110.stdout:2277: left oid 4 (ObjNum 719 snap 229 seq_num 719) 2024-09-14T03:45:30.837 INFO:tasks.rados.rados.0.smithi110.stdout:2276: finishing write tid 3 to smithi11075562-38 2024-09-14T03:45:30.837 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1569 (ObjNum 718 snap 229 seq_num 718) dirty exists 2024-09-14T03:45:30.837 INFO:tasks.rados.rados.0.smithi110.stdout:2276: left oid 38 (ObjNum 718 snap 229 seq_num 718) 2024-09-14T03:45:30.837 INFO:tasks.rados.rados.0.smithi110.stdout:2274: expect (ObjNum 592 snap 181 seq_num 592) 2024-09-14T03:45:30.942 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:45:30.942 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:45:30.942 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:45:30.943 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3, 2024-09-14T03:45:30.944 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 12 2024-09-14T03:45:30.944 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:45:30.944 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:45:30.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:30 smithi110 ceph-mon[88262]: osdmap e528: 8 total, 8 up, 8 in 2024-09-14T03:45:30.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:30 smithi110 ceph-mon[88262]: pgmap v421: 121 pgs: 121 active+clean; 251 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-14T03:45:30.944 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:30 smithi110 ceph-mon[86423]: osdmap e528: 8 total, 8 up, 8 in 2024-09-14T03:45:30.944 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:30 smithi110 ceph-mon[86423]: pgmap v421: 121 pgs: 121 active+clean; 251 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-14T03:45:31.189 INFO:tasks.rados.rados.0.smithi110.stdout:2272: finishing copy_from racing read to smithi11075562-21 2024-09-14T03:45:31.189 INFO:tasks.rados.rados.0.smithi110.stdout:2272: finishing copy_from to smithi11075562-21 2024-09-14T03:45:31.189 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 952 (ObjNum 696 snap 219 seq_num 696) dirty exists 2024-09-14T03:45:31.189 INFO:tasks.rados.rados.0.smithi110.stdout:2278: finishing write tid 1 to smithi11075562-42 2024-09-14T03:45:31.189 INFO:tasks.rados.rados.0.smithi110.stdout:2278: finishing write tid 2 to smithi11075562-42 2024-09-14T03:45:31.189 INFO:tasks.rados.rados.0.smithi110.stdout:2281: seq_num 721 ranges {4562944=81920} 2024-09-14T03:45:31.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:30 smithi195 ceph-mon[82388]: osdmap e528: 8 total, 8 up, 8 in 2024-09-14T03:45:31.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:30 smithi195 ceph-mon[82388]: pgmap v421: 121 pgs: 121 active+clean; 251 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-14T03:45:31.214 INFO:tasks.rados.rados.0.smithi110.stdout:2281: writing smithi11075562-23 from 4562944 to 4644864 tid 1 2024-09-14T03:45:31.214 INFO:tasks.rados.rados.0.smithi110.stdout:2272: done (7 left) 2024-09-14T03:45:31.214 INFO:tasks.rados.rados.0.smithi110.stdout:2274: done (6 left) 2024-09-14T03:45:31.214 INFO:tasks.rados.rados.0.smithi110.stdout:2276: done (5 left) 2024-09-14T03:45:31.214 INFO:tasks.rados.rados.0.smithi110.stdout:2277: done (4 left) 2024-09-14T03:45:31.214 INFO:tasks.rados.rados.0.smithi110.stdout:2279: done (3 left) 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:2282: read oid 28 snap -1 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:2282: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:2283: delete oid 35 current snap is 229 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:2278: finishing write tid 3 to smithi11075562-42 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1314 (ObjNum 720 snap 229 seq_num 720) dirty exists 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:2278: left oid 42 (ObjNum 720 snap 229 seq_num 720) 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:2280: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:2280: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:2280: finishing rollback tid 2 to smithi11075562-17 2024-09-14T03:45:31.215 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1136 (ObjNum 551 snap 167 seq_num 551) dirty exists 2024-09-14T03:45:31.216 INFO:tasks.rados.rados.0.smithi110.stdout:2278: done (4 left) 2024-09-14T03:45:31.216 INFO:tasks.rados.rados.0.smithi110.stdout:2280: done (3 left) 2024-09-14T03:45:31.216 INFO:tasks.rados.rados.0.smithi110.stdout:2283: done (2 left) 2024-09-14T03:45:31.216 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 229 2024-09-14T03:45:31.216 INFO:tasks.rados.rados.0.smithi110.stdout:2284: seq_num 723 ranges {2342912=114688} 2024-09-14T03:45:31.230 INFO:tasks.rados.rados.0.smithi110.stdout:2284: writing smithi11075562-49 from 2342912 to 2457600 tid 1 2024-09-14T03:45:31.230 INFO:tasks.rados.rados.0.smithi110.stdout:2285: rollback oid 34 current snap is 229 2024-09-14T03:45:31.230 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 228 2024-09-14T03:45:31.230 INFO:tasks.rados.rados.0.smithi110.stdout:2282: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "mon" 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "12/20 daemons upgraded", 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:45:31.237 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:45:31.238 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:45:31.246 INFO:tasks.rados.rados.0.smithi110.stdout:2281: finishing write tid 1 to smithi11075562-23 2024-09-14T03:45:31.246 INFO:tasks.rados.rados.0.smithi110.stdout:2281: finishing write tid 2 to smithi11075562-23 2024-09-14T03:45:31.246 INFO:tasks.rados.rados.0.smithi110.stdout:2281: finishing write tid 3 to smithi11075562-23 2024-09-14T03:45:31.246 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1481 (ObjNum 721 snap 229 seq_num 721) dirty exists 2024-09-14T03:45:31.246 INFO:tasks.rados.rados.0.smithi110.stdout:2281: left oid 23 (ObjNum 721 snap 229 seq_num 721) 2024-09-14T03:45:31.246 INFO:tasks.rados.rados.0.smithi110.stdout:2281: done (3 left) 2024-09-14T03:45:31.246 INFO:tasks.rados.rados.0.smithi110.stdout:2282: done (2 left) 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:2286: setattr oid 13 current snap is 229 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:2284: finishing write tid 1 to smithi11075562-49 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:2285: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:2285: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:2284: finishing write tid 2 to smithi11075562-49 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:2285: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1215 (ObjNum 553 snap 168 seq_num 553) dirty exists 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:2284: finishing write tid 3 to smithi11075562-49 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1318 (ObjNum 723 snap 229 seq_num 723) dirty exists 2024-09-14T03:45:31.247 INFO:tasks.rados.rados.0.smithi110.stdout:2284: left oid 49 (ObjNum 723 snap 229 seq_num 723) 2024-09-14T03:45:31.248 INFO:tasks.rados.rados.0.smithi110.stdout:2284: done (2 left) 2024-09-14T03:45:31.248 INFO:tasks.rados.rados.0.smithi110.stdout:2285: done (1 left) 2024-09-14T03:45:31.249 INFO:tasks.rados.rados.0.smithi110.stdout:2287: read oid 5 snap -1 2024-09-14T03:45:31.249 INFO:tasks.rados.rados.0.smithi110.stdout:2287: expect (ObjNum 697 snap 220 seq_num 697) 2024-09-14T03:45:31.249 INFO:tasks.rados.rados.0.smithi110.stdout:2288: snap_remove snap 227 2024-09-14T03:45:31.752 INFO:tasks.rados.rados.0.smithi110.stdout:2288: done (2 left) 2024-09-14T03:45:31.753 INFO:tasks.rados.rados.0.smithi110.stdout:2289: read oid 44 snap -1 2024-09-14T03:45:31.753 INFO:tasks.rados.rados.0.smithi110.stdout:2289: expect (ObjNum 712 snap 227 seq_num 712) 2024-09-14T03:45:31.753 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 229 2024-09-14T03:45:31.753 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1320 (ObjNum 550 snap 167 seq_num 550) dirty exists 2024-09-14T03:45:31.753 INFO:tasks.rados.rados.0.smithi110.stdout:2287: expect (ObjNum 697 snap 220 seq_num 697) 2024-09-14T03:45:31.920 INFO:tasks.rados.rados.0.smithi110.stdout:2289: expect (ObjNum 712 snap 227 seq_num 712) 2024-09-14T03:45:31.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:45:31] "GET /metrics HTTP/1.1" 200 36253 "" "Prometheus/2.51.0" 2024-09-14T03:45:31.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[86423]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:31.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[86423]: from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:31.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[86423]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:31.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[86423]: osdmap e529: 8 total, 8 up, 8 in 2024-09-14T03:45:31.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3750365951' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:31.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[88262]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:31.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[88262]: from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:31.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[88262]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:31.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[88262]: osdmap e529: 8 total, 8 up, 8 in 2024-09-14T03:45:31.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:31 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3750365951' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:31.980 INFO:tasks.rados.rados.0.smithi110.stdout:2290: seq_num 724 ranges {4644864=131072} 2024-09-14T03:45:32.005 INFO:tasks.rados.rados.0.smithi110.stdout:2290: writing smithi11075562-23 from 4644864 to 4775936 tid 1 2024-09-14T03:45:32.005 INFO:tasks.rados.rados.0.smithi110.stdout:2286: done (3 left) 2024-09-14T03:45:32.005 INFO:tasks.rados.rados.0.smithi110.stdout:2287: done (2 left) 2024-09-14T03:45:32.005 INFO:tasks.rados.rados.0.smithi110.stdout:2289: done (1 left) 2024-09-14T03:45:32.005 INFO:tasks.rados.rados.0.smithi110.stdout:2291: delete oid 37 current snap is 229 2024-09-14T03:45:32.008 INFO:tasks.rados.rados.0.smithi110.stdout:2290: finishing write tid 1 to smithi11075562-23 2024-09-14T03:45:32.009 INFO:tasks.rados.rados.0.smithi110.stdout:2291: done (1 left) 2024-09-14T03:45:32.010 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 229 2024-09-14T03:45:32.010 INFO:tasks.rados.rados.0.smithi110.stdout:2290: finishing write tid 2 to smithi11075562-23 2024-09-14T03:45:32.010 INFO:tasks.rados.rados.0.smithi110.stdout:2292: seq_num 726 ranges {4423680=163840} 2024-09-14T03:45:32.034 INFO:tasks.rados.rados.0.smithi110.stdout:2292: writing smithi11075562-5 from 4423680 to 4587520 tid 1 2024-09-14T03:45:32.034 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 229 2024-09-14T03:45:32.034 INFO:tasks.rados.rados.0.smithi110.stdout:2293: seq_num 727 ranges {5472256=180224} 2024-09-14T03:45:32.063 INFO:tasks.rados.rados.0.smithi110.stdout:2293: writing smithi11075562-13 from 5472256 to 5652480 tid 1 2024-09-14T03:45:32.063 INFO:tasks.rados.rados.0.smithi110.stdout:2294: snap_remove snap 216 2024-09-14T03:45:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:31 smithi195 ceph-mon[82388]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:31 smithi195 ceph-mon[82388]: from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:31 smithi195 ceph-mon[82388]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:31 smithi195 ceph-mon[82388]: osdmap e529: 8 total, 8 up, 8 in 2024-09-14T03:45:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:31 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3750365951' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:2294: done (3 left) 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:2295: setattr oid 26 current snap is 229 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:2290: finishing write tid 3 to smithi11075562-23 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1483 (ObjNum 724 snap 229 seq_num 724) dirty exists 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:2290: left oid 23 (ObjNum 724 snap 229 seq_num 724) 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:2292: finishing write tid 1 to smithi11075562-5 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:2292: finishing write tid 2 to smithi11075562-5 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:2292: finishing write tid 3 to smithi11075562-5 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1616 (ObjNum 726 snap 229 seq_num 726) dirty exists 2024-09-14T03:45:32.763 INFO:tasks.rados.rados.0.smithi110.stdout:2292: left oid 5 (ObjNum 726 snap 229 seq_num 726) 2024-09-14T03:45:32.764 INFO:tasks.rados.rados.0.smithi110.stdout:2293: finishing write tid 1 to smithi11075562-13 2024-09-14T03:45:32.764 INFO:tasks.rados.rados.0.smithi110.stdout:2293: finishing write tid 2 to smithi11075562-13 2024-09-14T03:45:32.764 INFO:tasks.rados.rados.0.smithi110.stdout:2293: finishing write tid 3 to smithi11075562-13 2024-09-14T03:45:32.764 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1328 (ObjNum 727 snap 229 seq_num 727) dirty exists 2024-09-14T03:45:32.764 INFO:tasks.rados.rados.0.smithi110.stdout:2293: left oid 13 (ObjNum 727 snap 229 seq_num 727) 2024-09-14T03:45:32.765 INFO:tasks.rados.rados.0.smithi110.stdout:2290: done (3 left) 2024-09-14T03:45:32.765 INFO:tasks.rados.rados.0.smithi110.stdout:2292: done (2 left) 2024-09-14T03:45:32.765 INFO:tasks.rados.rados.0.smithi110.stdout:2293: done (1 left) 2024-09-14T03:45:32.765 INFO:tasks.rados.rados.0.smithi110.stdout:2296: snap_create 2024-09-14T03:45:32.772 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 788 (ObjNum 81 snap 0 seq_num 1818713172) dirty exists 2024-09-14T03:45:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:32 smithi110 ceph-mon[86423]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:32 smithi110 ceph-mon[86423]: osdmap e530: 8 total, 8 up, 8 in 2024-09-14T03:45:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:32 smithi110 ceph-mon[86423]: pgmap v424: 121 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 119 active+clean; 251 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-14T03:45:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:32 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:33.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:32 smithi110 ceph-mon[88262]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:33.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:32 smithi110 ceph-mon[88262]: osdmap e530: 8 total, 8 up, 8 in 2024-09-14T03:45:33.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:32 smithi110 ceph-mon[88262]: pgmap v424: 121 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 119 active+clean; 251 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-14T03:45:33.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:32 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:33.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:32 smithi195 ceph-mon[82388]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:32 smithi195 ceph-mon[82388]: osdmap e530: 8 total, 8 up, 8 in 2024-09-14T03:45:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:32 smithi195 ceph-mon[82388]: pgmap v424: 121 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 119 active+clean; 251 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-14T03:45:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:32 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:33.735 INFO:tasks.rados.rados.0.smithi110.stdout:2295: done (1 left) 2024-09-14T03:45:33.735 INFO:tasks.rados.rados.0.smithi110.stdout:2296: done (0 left) 2024-09-14T03:45:33.735 INFO:tasks.rados.rados.0.smithi110.stdout:2297: snap_remove snap 228 2024-09-14T03:45:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:33 smithi110 ceph-mon[86423]: osdmap e531: 8 total, 8 up, 8 in 2024-09-14T03:45:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:33 smithi110 ceph-mon[86423]: osdmap e532: 8 total, 8 up, 8 in 2024-09-14T03:45:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:33 smithi110 ceph-mon[88262]: osdmap e531: 8 total, 8 up, 8 in 2024-09-14T03:45:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:33 smithi110 ceph-mon[88262]: osdmap e532: 8 total, 8 up, 8 in 2024-09-14T03:45:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:33 smithi195 ceph-mon[82388]: osdmap e531: 8 total, 8 up, 8 in 2024-09-14T03:45:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:33 smithi195 ceph-mon[82388]: osdmap e532: 8 total, 8 up, 8 in 2024-09-14T03:45:34.737 INFO:tasks.rados.rados.0.smithi110.stdout:2297: done (0 left) 2024-09-14T03:45:34.737 INFO:tasks.rados.rados.0.smithi110.stdout:2298: read oid 8 snap -1 2024-09-14T03:45:34.738 INFO:tasks.rados.rados.0.smithi110.stdout:2298: expect deleted 2024-09-14T03:45:34.738 INFO:tasks.rados.rados.0.smithi110.stdout:2299: read oid 35 snap -1 2024-09-14T03:45:34.738 INFO:tasks.rados.rados.0.smithi110.stdout:2299: expect deleted 2024-09-14T03:45:34.738 INFO:tasks.rados.rados.0.smithi110.stdout:2300: read oid 6 snap -1 2024-09-14T03:45:34.738 INFO:tasks.rados.rados.0.smithi110.stdout:2300: expect deleted 2024-09-14T03:45:34.738 INFO:tasks.rados.rados.0.smithi110.stdout:2301: snap_remove snap 225 2024-09-14T03:45:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:34 smithi110 ceph-mon[86423]: pgmap v427: 121 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 119 active+clean; 257 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-14T03:45:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:34 smithi110 ceph-mon[86423]: osdmap e533: 8 total, 8 up, 8 in 2024-09-14T03:45:35.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:34 smithi110 ceph-mon[88262]: pgmap v427: 121 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 119 active+clean; 257 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-14T03:45:35.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:34 smithi110 ceph-mon[88262]: osdmap e533: 8 total, 8 up, 8 in 2024-09-14T03:45:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:34 smithi195 ceph-mon[82388]: pgmap v427: 121 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 119 active+clean; 257 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-14T03:45:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:34 smithi195 ceph-mon[82388]: osdmap e533: 8 total, 8 up, 8 in 2024-09-14T03:45:35.739 INFO:tasks.rados.rados.0.smithi110.stdout:2301: done (3 left) 2024-09-14T03:45:35.739 INFO:tasks.rados.rados.0.smithi110.stdout:2302: snap_create 2024-09-14T03:45:36.745 INFO:tasks.rados.rados.0.smithi110.stdout:2298: done (3 left) 2024-09-14T03:45:36.745 INFO:tasks.rados.rados.0.smithi110.stdout:2299: done (2 left) 2024-09-14T03:45:36.746 INFO:tasks.rados.rados.0.smithi110.stdout:2300: done (1 left) 2024-09-14T03:45:36.746 INFO:tasks.rados.rados.0.smithi110.stdout:2302: done (0 left) 2024-09-14T03:45:36.746 INFO:tasks.rados.rados.0.smithi110.stdout:2303: snap_remove snap 204 2024-09-14T03:45:37.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:36 smithi195 ceph-mon[82388]: osdmap e534: 8 total, 8 up, 8 in 2024-09-14T03:45:37.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:36 smithi195 ceph-mon[82388]: pgmap v430: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 255 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.6 MiB/s wr, 19 op/s 2024-09-14T03:45:37.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:36 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:45:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:36 smithi110 ceph-mon[88262]: osdmap e534: 8 total, 8 up, 8 in 2024-09-14T03:45:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:36 smithi110 ceph-mon[88262]: pgmap v430: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 255 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.6 MiB/s wr, 19 op/s 2024-09-14T03:45:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:36 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:45:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:36 smithi110 ceph-mon[86423]: osdmap e534: 8 total, 8 up, 8 in 2024-09-14T03:45:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:36 smithi110 ceph-mon[86423]: pgmap v430: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 255 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.6 MiB/s wr, 19 op/s 2024-09-14T03:45:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:36 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:45:37.749 INFO:tasks.rados.rados.0.smithi110.stdout:2303: done (0 left) 2024-09-14T03:45:37.749 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 231 2024-09-14T03:45:37.749 INFO:tasks.rados.rados.0.smithi110.stdout:2304: seq_num 728 ranges {0=196608} 2024-09-14T03:45:37.750 INFO:tasks.rados.rados.0.smithi110.stdout:2304: writing smithi11075562-6 from 0 to 196608 tid 1 2024-09-14T03:45:37.750 INFO:tasks.rados.rados.0.smithi110.stdout:2305: copy_from oid 20 from oid 45 current snap is 231 2024-09-14T03:45:37.751 INFO:tasks.rados.rados.0.smithi110.stdout:2306: rollback oid 39 current snap is 231 2024-09-14T03:45:37.751 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 217 2024-09-14T03:45:37.751 INFO:tasks.rados.rados.0.smithi110.stdout:2307: read oid 30 snap 229 2024-09-14T03:45:37.751 INFO:tasks.rados.rados.0.smithi110.stdout:2307: expect (ObjNum 687 snap 215 seq_num 687) 2024-09-14T03:45:37.751 INFO:tasks.rados.rados.0.smithi110.stdout:2308: read oid 26 snap 226 2024-09-14T03:45:37.751 INFO:tasks.rados.rados.0.smithi110.stdout:2308: expect deleted 2024-09-14T03:45:37.751 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 231 2024-09-14T03:45:37.751 INFO:tasks.rados.rados.0.smithi110.stdout:2309: seq_num 729 ranges {2392064=212992} 2024-09-14T03:45:37.767 INFO:tasks.rados.rados.0.smithi110.stdout:2309: writing smithi11075562-32 from 2392064 to 2605056 tid 1 2024-09-14T03:45:37.767 INFO:tasks.rados.rados.0.smithi110.stdout:2310: read oid 3 snap -1 2024-09-14T03:45:37.767 INFO:tasks.rados.rados.0.smithi110.stdout:2310: expect deleted 2024-09-14T03:45:37.767 INFO:tasks.rados.rados.0.smithi110.stdout:2311: snap_create 2024-09-14T03:45:37.767 INFO:tasks.rados.rados.0.smithi110.stdout:2304: finishing write tid 1 to smithi11075562-6 2024-09-14T03:45:37.767 INFO:tasks.rados.rados.0.smithi110.stdout:2304: finishing write tid 2 to smithi11075562-6 2024-09-14T03:45:37.768 INFO:tasks.rados.rados.0.smithi110.stdout:2304: finishing write tid 3 to smithi11075562-6 2024-09-14T03:45:37.768 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1244 (ObjNum 728 snap 231 seq_num 728) dirty exists 2024-09-14T03:45:37.768 INFO:tasks.rados.rados.0.smithi110.stdout:2304: left oid 6 (ObjNum 728 snap 231 seq_num 728) 2024-09-14T03:45:37.768 INFO:tasks.rados.rados.0.smithi110.stdout:2306: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:45:37.768 INFO:tasks.rados.rados.0.smithi110.stdout:2306: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:45:37.768 INFO:tasks.rados.rados.0.smithi110.stdout:2306: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:45:37.768 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1491 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:45:37.772 INFO:tasks.rados.rados.0.smithi110.stdout:2305: finishing copy_from racing read to smithi11075562-20 2024-09-14T03:45:37.774 INFO:tasks.rados.rados.0.smithi110.stdout:2305: finishing copy_from to smithi11075562-20 2024-09-14T03:45:37.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1562 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:45:37.792 INFO:tasks.rados.rados.0.smithi110.stdout:2307: expect (ObjNum 687 snap 215 seq_num 687) 2024-09-14T03:45:37.996 INFO:tasks.rados.rados.0.smithi110.stdout:2309: finishing write tid 1 to smithi11075562-32 2024-09-14T03:45:37.996 INFO:tasks.rados.rados.0.smithi110.stdout:2309: finishing write tid 2 to smithi11075562-32 2024-09-14T03:45:37.996 INFO:tasks.rados.rados.0.smithi110.stdout:2309: finishing write tid 3 to smithi11075562-32 2024-09-14T03:45:37.997 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1409 (ObjNum 729 snap 231 seq_num 729) dirty exists 2024-09-14T03:45:37.997 INFO:tasks.rados.rados.0.smithi110.stdout:2309: left oid 32 (ObjNum 729 snap 231 seq_num 729) 2024-09-14T03:45:38.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:37 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:45:38.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:37 smithi110 ceph-mon[88262]: Upgrade: osd.7 is safe to restart 2024-09-14T03:45:38.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:37 smithi110 ceph-mon[88262]: osdmap e535: 8 total, 8 up, 8 in 2024-09-14T03:45:38.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:37 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:45:38.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:37 smithi110 ceph-mon[86423]: Upgrade: osd.7 is safe to restart 2024-09-14T03:45:38.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:37 smithi110 ceph-mon[86423]: osdmap e535: 8 total, 8 up, 8 in 2024-09-14T03:45:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:37 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:45:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:37 smithi195 ceph-mon[82388]: Upgrade: osd.7 is safe to restart 2024-09-14T03:45:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:37 smithi195 ceph-mon[82388]: osdmap e535: 8 total, 8 up, 8 in 2024-09-14T03:45:38.736 INFO:tasks.rados.rados.0.smithi110.stdout:2304: done (7 left) 2024-09-14T03:45:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:2305: done (6 left) 2024-09-14T03:45:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:2306: done (5 left) 2024-09-14T03:45:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:2307: done (4 left) 2024-09-14T03:45:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:2308: done (3 left) 2024-09-14T03:45:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:2309: done (2 left) 2024-09-14T03:45:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:2310: done (1 left) 2024-09-14T03:45:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:2311: done (0 left) 2024-09-14T03:45:38.737 INFO:tasks.rados.rados.0.smithi110.stdout:2312: delete oid 37 current snap is 232 2024-09-14T03:45:38.741 INFO:tasks.rados.rados.0.smithi110.stdout:2312: done (0 left) 2024-09-14T03:45:38.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 232 2024-09-14T03:45:38.741 INFO:tasks.rados.rados.0.smithi110.stdout:2313: seq_num 731 ranges {49152=245760} 2024-09-14T03:45:38.742 INFO:tasks.rados.rados.0.smithi110.stdout:2313: writing smithi11075562-4 from 49152 to 294912 tid 1 2024-09-14T03:45:38.742 INFO:tasks.rados.rados.0.smithi110.stdout:2314: snap_create 2024-09-14T03:45:38.747 INFO:tasks.rados.rados.0.smithi110.stdout:2313: finishing write tid 1 to smithi11075562-4 2024-09-14T03:45:38.747 INFO:tasks.rados.rados.0.smithi110.stdout:2313: finishing write tid 2 to smithi11075562-4 2024-09-14T03:45:38.748 INFO:tasks.rados.rados.0.smithi110.stdout:2313: finishing write tid 3 to smithi11075562-4 2024-09-14T03:45:38.748 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1217 (ObjNum 731 snap 232 seq_num 731) dirty exists 2024-09-14T03:45:38.748 INFO:tasks.rados.rados.0.smithi110.stdout:2313: left oid 4 (ObjNum 731 snap 232 seq_num 731) 2024-09-14T03:45:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:38 smithi195 ceph-mon[82388]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T03:45:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:38 smithi195 ceph-mon[82388]: pgmap v433: 121 pgs: 1 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 119 active+clean; 250 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 148 KiB/s wr, 13 op/s 2024-09-14T03:45:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:38 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:38 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:45:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:38 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:38.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:38 smithi195 ceph-mon[82388]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T03:45:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[88262]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T03:45:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[88262]: pgmap v433: 121 pgs: 1 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 119 active+clean; 250 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 148 KiB/s wr, 13 op/s 2024-09-14T03:45:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:45:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:39.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[88262]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T03:45:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[86423]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T03:45:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[86423]: pgmap v433: 121 pgs: 1 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 119 active+clean; 250 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 148 KiB/s wr, 13 op/s 2024-09-14T03:45:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:45:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:38 smithi110 ceph-mon[86423]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T03:45:39.739 INFO:tasks.rados.rados.0.smithi110.stdout:2313: done (1 left) 2024-09-14T03:45:39.739 INFO:tasks.rados.rados.0.smithi110.stdout:2314: done (0 left) 2024-09-14T03:45:39.739 INFO:tasks.rados.rados.0.smithi110.stdout:2315: read oid 39 snap 230 2024-09-14T03:45:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:2315: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:45:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:2316: rmattr oid 42 current snap is 233 2024-09-14T03:45:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:2316: done (1 left) 2024-09-14T03:45:39.740 INFO:tasks.rados.rados.0.smithi110.stdout:2317: delete oid 7 current snap is 233 2024-09-14T03:45:39.753 INFO:tasks.rados.rados.0.smithi110.stdout:2317: done (1 left) 2024-09-14T03:45:39.753 INFO:tasks.rados.rados.0.smithi110.stdout:2318: copy_from oid 27 from oid 40 current snap is 233 2024-09-14T03:45:39.753 INFO:tasks.rados.rados.0.smithi110.stdout:2319: rollback oid 45 current snap is 233 2024-09-14T03:45:39.753 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 214 2024-09-14T03:45:39.753 INFO:tasks.rados.rados.0.smithi110.stdout:2320: copy_from oid 50 from oid 3 current snap is 233 2024-09-14T03:45:39.754 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 233 2024-09-14T03:45:39.754 INFO:tasks.rados.rados.0.smithi110.stdout:2321: seq_num 733 ranges {6946816=278528} 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:2321: writing smithi11075562-31 from 6946816 to 7225344 tid 1 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 233 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:2320: finishing copy_from to smithi11075562-50 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:2320: got expected ENOENT (src dne) 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:2320: finishing copy_from racing read to smithi11075562-50 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:2319: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:2319: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:2319: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:45:39.795 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 1643 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:45:39.796 INFO:tasks.rados.rados.0.smithi110.stdout:2318: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:45:39.796 INFO:tasks.rados.rados.0.smithi110.stdout:2318: finishing copy_from to smithi11075562-27 2024-09-14T03:45:39.796 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1178 (ObjNum 693 snap 216 seq_num 693) dirty exists 2024-09-14T03:45:39.796 INFO:tasks.rados.rados.0.smithi110.stdout:2322: seq_num 734 ranges {0=294912} 2024-09-14T03:45:39.797 INFO:tasks.rados.rados.0.smithi110.stdout:2322: writing smithi11075562-41 from 0 to 294912 tid 1 2024-09-14T03:45:39.797 INFO:tasks.rados.rados.0.smithi110.stdout:2318: done (5 left) 2024-09-14T03:45:39.797 INFO:tasks.rados.rados.0.smithi110.stdout:2319: done (4 left) 2024-09-14T03:45:39.797 INFO:tasks.rados.rados.0.smithi110.stdout:2320: done (3 left) 2024-09-14T03:45:39.798 INFO:tasks.rados.rados.0.smithi110.stdout:2323: read oid 12 snap -1 2024-09-14T03:45:39.798 INFO:tasks.rados.rados.0.smithi110.stdout:2323: expect deleted 2024-09-14T03:45:39.798 INFO:tasks.rados.rados.0.smithi110.stdout:2324: rmattr oid 21 current snap is 233 2024-09-14T03:45:39.798 INFO:tasks.rados.rados.0.smithi110.stdout:2324: done (4 left) 2024-09-14T03:45:39.798 INFO:tasks.rados.rados.0.smithi110.stdout:2325: snap_create 2024-09-14T03:45:39.802 INFO:tasks.rados.rados.0.smithi110.stdout:2321: finishing write tid 1 to smithi11075562-31 2024-09-14T03:45:39.803 INFO:tasks.rados.rados.0.smithi110.stdout:2321: finishing write tid 2 to smithi11075562-31 2024-09-14T03:45:39.804 INFO:tasks.rados.rados.0.smithi110.stdout:2321: finishing write tid 3 to smithi11075562-31 2024-09-14T03:45:39.804 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1219 (ObjNum 733 snap 233 seq_num 733) dirty exists 2024-09-14T03:45:39.804 INFO:tasks.rados.rados.0.smithi110.stdout:2321: left oid 31 (ObjNum 733 snap 233 seq_num 733) 2024-09-14T03:45:39.815 INFO:tasks.rados.rados.0.smithi110.stdout:2315: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:45:39.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:39 smithi195 ceph-mon[82388]: Upgrade: Updating osd.7 2024-09-14T03:45:39.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:39 smithi195 ceph-mon[82388]: Deploying daemon osd.7 on smithi195 2024-09-14T03:45:39.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:39 smithi195 ceph-mon[82388]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T03:45:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:39 smithi110 ceph-mon[88262]: Upgrade: Updating osd.7 2024-09-14T03:45:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:39 smithi110 ceph-mon[88262]: Deploying daemon osd.7 on smithi195 2024-09-14T03:45:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:39 smithi110 ceph-mon[88262]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T03:45:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:39 smithi110 ceph-mon[86423]: Upgrade: Updating osd.7 2024-09-14T03:45:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:39 smithi110 ceph-mon[86423]: Deploying daemon osd.7 on smithi195 2024-09-14T03:45:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:39 smithi110 ceph-mon[86423]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T03:45:40.524 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:45:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:45:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:45:40.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:40 smithi195 ceph-mon[82388]: pgmap v436: 121 pgs: 1 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 119 active+clean; 244 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 156 KiB/s wr, 3 op/s 2024-09-14T03:45:40.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:40 smithi195 systemd[1]: Stopping Ceph osd.7 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:45:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:40 smithi110 ceph-mon[88262]: pgmap v436: 121 pgs: 1 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 119 active+clean; 244 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 156 KiB/s wr, 3 op/s 2024-09-14T03:45:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:40 smithi110 ceph-mon[86423]: pgmap v436: 121 pgs: 1 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 119 active+clean; 244 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 156 KiB/s wr, 3 op/s 2024-09-14T03:45:41.247 INFO:tasks.rados.rados.0.smithi110.stdout:2322: finishing write tid 1 to smithi11075562-41 2024-09-14T03:45:41.247 INFO:tasks.rados.rados.0.smithi110.stdout:2322: finishing write tid 2 to smithi11075562-41 2024-09-14T03:45:41.247 INFO:tasks.rados.rados.0.smithi110.stdout:2322: finishing write tid 3 to smithi11075562-41 2024-09-14T03:45:41.247 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1493 (ObjNum 734 snap 233 seq_num 734) dirty exists 2024-09-14T03:45:41.247 INFO:tasks.rados.rados.0.smithi110.stdout:2322: left oid 41 (ObjNum 734 snap 233 seq_num 734) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2315: done (4 left) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2321: done (3 left) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2322: done (2 left) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2323: done (1 left) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2325: done (0 left) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2326: read oid 44 snap 214 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2326: expect (ObjNum 553 snap 168 seq_num 553) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2327: read oid 43 snap -1 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2327: expect (ObjNum 613 snap 188 seq_num 613) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2328: read oid 11 snap -1 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2328: expect deleted 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2329: read oid 45 snap 231 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2329: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:45:41.248 INFO:tasks.rados.rados.0.smithi110.stdout:2330: delete oid 34 current snap is 234 2024-09-14T03:45:41.256 INFO:tasks.rados.rados.0.smithi110.stdout:2328: done (4 left) 2024-09-14T03:45:41.256 INFO:tasks.rados.rados.0.smithi110.stdout:2330: done (3 left) 2024-09-14T03:45:41.256 INFO:tasks.rados.rados.0.smithi110.stdout:2331: read oid 9 snap -1 2024-09-14T03:45:41.256 INFO:tasks.rados.rados.0.smithi110.stdout:2331: expect (ObjNum 654 snap 202 seq_num 654) 2024-09-14T03:45:41.256 INFO:tasks.rados.rados.0.smithi110.stdout:2332: snap_create 2024-09-14T03:45:41.266 INFO:tasks.rados.rados.0.smithi110.stdout:2329: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:45:41.340 INFO:tasks.rados.rados.0.smithi110.stdout:2327: expect (ObjNum 613 snap 188 seq_num 613) 2024-09-14T03:45:41.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[53109]: 2024-09-14T03:45:40.960+0000 7f50602f9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:45:41.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[53109]: 2024-09-14T03:45:40.960+0000 7f50602f9640 -1 osd.7 538 *** Got signal Terminated *** 2024-09-14T03:45:41.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[53109]: 2024-09-14T03:45:40.960+0000 7f50602f9640 -1 osd.7 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:45:41.563 INFO:tasks.rados.rados.0.smithi110.stdout:2331: expect (ObjNum 654 snap 202 seq_num 654) 2024-09-14T03:45:41.597 INFO:tasks.rados.rados.0.smithi110.stdout:2326: expect (ObjNum 553 snap 168 seq_num 553) 2024-09-14T03:45:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:41 smithi110 ceph-mon[88262]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T03:45:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:41 smithi110 ceph-mon[88262]: osd.7 marked itself down and dead 2024-09-14T03:45:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:41 smithi110 ceph-mon[86423]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T03:45:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:41 smithi110 ceph-mon[86423]: osd.7 marked itself down and dead 2024-09-14T03:45:41.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:45:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:45:41] "GET /metrics HTTP/1.1" 200 36253 "" "Prometheus/2.51.0" 2024-09-14T03:45:41.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:45:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:45:41.780+0000 7f9409080640 -1 calc_pg_upmaps abort due to max <= 0 2024-09-14T03:45:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:41 smithi195 ceph-mon[82388]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T03:45:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:41 smithi195 ceph-mon[82388]: osd.7 marked itself down and dead 2024-09-14T03:45:42.347 INFO:tasks.rados.rados.0.smithi110.stdout:2326: done (4 left) 2024-09-14T03:45:42.347 INFO:tasks.rados.rados.0.smithi110.stdout:2327: done (3 left) 2024-09-14T03:45:42.347 INFO:tasks.rados.rados.0.smithi110.stdout:2329: done (2 left) 2024-09-14T03:45:42.347 INFO:tasks.rados.rados.0.smithi110.stdout:2331: done (1 left) 2024-09-14T03:45:42.347 INFO:tasks.rados.rados.0.smithi110.stdout:2332: done (0 left) 2024-09-14T03:45:42.347 INFO:tasks.rados.rados.0.smithi110.stdout:2333: read oid 46 snap 234 2024-09-14T03:45:42.347 INFO:tasks.rados.rados.0.smithi110.stdout:2333: expect (ObjNum 704 snap 225 seq_num 704) 2024-09-14T03:45:42.347 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 235 2024-09-14T03:45:42.348 INFO:tasks.rados.rados.0.smithi110.stdout:2334: seq_num 736 ranges {2605056=335872} 2024-09-14T03:45:42.362 INFO:tasks.rados.rados.0.smithi110.stdout:2334: writing smithi11075562-32 from 2605056 to 2940928 tid 1 2024-09-14T03:45:42.363 INFO:tasks.rados.rados.0.smithi110.stdout:2335: read oid 31 snap -1 2024-09-14T03:45:42.363 INFO:tasks.rados.rados.0.smithi110.stdout:2335: expect (ObjNum 733 snap 233 seq_num 733) 2024-09-14T03:45:42.363 INFO:tasks.rados.rados.0.smithi110.stdout:2336: read oid 47 snap -1 2024-09-14T03:45:42.363 INFO:tasks.rados.rados.0.smithi110.stdout:2336: expect (ObjNum 674 snap 209 seq_num 674) 2024-09-14T03:45:42.363 INFO:tasks.rados.rados.0.smithi110.stdout:2337: setattr oid 5 current snap is 235 2024-09-14T03:45:42.364 INFO:tasks.rados.rados.0.smithi110.stdout:2338: snap_remove snap 230 2024-09-14T03:45:42.630 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:42 smithi195 podman[100490]: 2024-09-14 03:45:42.326817545 +0000 UTC m=+1.470857031 container died e5b6950370c81209ccf9e2d628d55b77121f88b771e5514730c4f1defbdb5800 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-14T03:45:42.778 INFO:tasks.rados.rados.0.smithi110.stdout:2334: finishing write tid 1 to smithi11075562-32 2024-09-14T03:45:42.778 INFO:tasks.rados.rados.0.smithi110.stdout:2334: finishing write tid 2 to smithi11075562-32 2024-09-14T03:45:42.778 INFO:tasks.rados.rados.0.smithi110.stdout:2334: finishing write tid 3 to smithi11075562-32 2024-09-14T03:45:42.778 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1412 (ObjNum 736 snap 235 seq_num 736) dirty exists 2024-09-14T03:45:42.779 INFO:tasks.rados.rados.0.smithi110.stdout:2334: left oid 32 (ObjNum 736 snap 235 seq_num 736) 2024-09-14T03:45:42.779 INFO:tasks.rados.rados.0.smithi110.stdout:2336: expect (ObjNum 674 snap 209 seq_num 674) 2024-09-14T03:45:42.907 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1645 (ObjNum 726 snap 229 seq_num 726) dirty exists 2024-09-14T03:45:42.907 INFO:tasks.rados.rados.0.smithi110.stdout:2333: expect (ObjNum 704 snap 225 seq_num 704) 2024-09-14T03:45:42.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:42 smithi195 podman[100490]: 2024-09-14 03:45:42.711530045 +0000 UTC m=+1.855569521 container cleanup e5b6950370c81209ccf9e2d628d55b77121f88b771e5514730c4f1defbdb5800 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:45:42.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:42 smithi195 bash[100490]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7 2024-09-14T03:45:42.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: osdmap e540: 8 total, 7 up, 8 in 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: pgmap v439: 121 pgs: 19 stale+active+clean, 1 active+clean+snaptrim_wait, 101 active+clean; 238 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 160 KiB/s wr, 3 op/s 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]} : dispatch 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]} : dispatch 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]} : dispatch 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]} : dispatch 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]} : dispatch 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]} : dispatch 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]} : dispatch 2024-09-14T03:45:42.960 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"} : dispatch 2024-09-14T03:45:42.961 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"}]': finished 2024-09-14T03:45:42.962 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:42 smithi195 ceph-mon[82388]: osdmap e541: 8 total, 7 up, 8 in 2024-09-14T03:45:42.999 INFO:tasks.rados.rados.0.smithi110.stdout:2335: expect (ObjNum 733 snap 233 seq_num 733) 2024-09-14T03:45:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:45:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-14T03:45:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: osdmap e540: 8 total, 7 up, 8 in 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: pgmap v439: 121 pgs: 19 stale+active+clean, 1 active+clean+snaptrim_wait, 101 active+clean; 238 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 160 KiB/s wr, 3 op/s 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]} : dispatch 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]} : dispatch 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]} : dispatch 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"} : dispatch 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]} : dispatch 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]} : dispatch 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]} : dispatch 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:45:43.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: osdmap e540: 8 total, 7 up, 8 in 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: pgmap v439: 121 pgs: 19 stale+active+clean, 1 active+clean+snaptrim_wait, 101 active+clean; 238 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 160 KiB/s wr, 3 op/s 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"} : dispatch 2024-09-14T03:45:43.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"}]': finished 2024-09-14T03:45:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[86423]: osdmap e541: 8 total, 7 up, 8 in 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"} : dispatch 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1", "id": [0, 7]}]': finished 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [0, 7]}]': finished 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2, 6, 1]}]': finished 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [0, 2]}]': finished 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [4, 7]}]': finished 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1, 5, 2]}]': finished 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [5, 7]}]': finished 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [0, 7]}]': finished 2024-09-14T03:45:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.e", "id": [5, 3]}]': finished 2024-09-14T03:45:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.5"}]': finished 2024-09-14T03:45:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:42 smithi110 ceph-mon[88262]: osdmap e541: 8 total, 7 up, 8 in 2024-09-14T03:45:43.337 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:42 smithi195 podman[100516]: 2024-09-14 03:45:42.996772214 +0000 UTC m=+0.671701933 container remove e5b6950370c81209ccf9e2d628d55b77121f88b771e5514730c4f1defbdb5800 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=quincy-eb16b97, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-14T03:45:43.651 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:43 smithi195 podman[100606]: 2024-09-14 03:45:43.23893939 +0000 UTC m=+0.016153224 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:45:43.651 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:43 smithi195 podman[100606]: 2024-09-14 03:45:43.347689819 +0000 UTC m=+0.124903643 container create 7b56745b654547d439d829f9bd388724adf53a06c52a45712c20cf97275d50dd (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-a278231, io.buildah.version=1.37.2) 2024-09-14T03:45:43.651 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:43 smithi195 podman[100606]: 2024-09-14 03:45:43.566751147 +0000 UTC m=+0.343964975 container init 7b56745b654547d439d829f9bd388724adf53a06c52a45712c20cf97275d50dd (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-14T03:45:43.651 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:43 smithi195 podman[100606]: 2024-09-14 03:45:43.570044246 +0000 UTC m=+0.347258070 container start 7b56745b654547d439d829f9bd388724adf53a06c52a45712c20cf97275d50dd (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T03:45:43.651 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:43 smithi195 podman[100606]: 2024-09-14 03:45:43.650264346 +0000 UTC m=+0.427478179 container attach 7b56745b654547d439d829f9bd388724adf53a06c52a45712c20cf97275d50dd (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T03:45:43.667 INFO:tasks.rados.rados.0.smithi110.stdout:2333: done (5 left) 2024-09-14T03:45:43.667 INFO:tasks.rados.rados.0.smithi110.stdout:2334: done (4 left) 2024-09-14T03:45:43.667 INFO:tasks.rados.rados.0.smithi110.stdout:2335: done (3 left) 2024-09-14T03:45:43.668 INFO:tasks.rados.rados.0.smithi110.stdout:2336: done (2 left) 2024-09-14T03:45:43.668 INFO:tasks.rados.rados.0.smithi110.stdout:2337: done (1 left) 2024-09-14T03:45:43.668 INFO:tasks.rados.rados.0.smithi110.stdout:2338: done (0 left) 2024-09-14T03:45:43.668 INFO:tasks.rados.rados.0.smithi110.stdout:2339: read oid 24 snap -1 2024-09-14T03:45:43.668 INFO:tasks.rados.rados.0.smithi110.stdout:2339: expect deleted 2024-09-14T03:45:43.668 INFO:tasks.rados.rados.0.smithi110.stdout:2340: delete oid 9 current snap is 235 2024-09-14T03:45:43.669 INFO:tasks.rados.rados.0.smithi110.stdout:2340: done (1 left) 2024-09-14T03:45:43.669 INFO:tasks.rados.rados.0.smithi110.stdout:2341: snap_remove snap 210 2024-09-14T03:45:43.738 INFO:tasks.rados.rados.0.smithi110.stdout:2341: done (1 left) 2024-09-14T03:45:43.738 INFO:tasks.rados.rados.0.smithi110.stdout:2342: rollback oid 44 current snap is 235 2024-09-14T03:45:43.738 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 44 to 234 2024-09-14T03:45:43.738 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 235 2024-09-14T03:45:43.738 INFO:tasks.rados.rados.0.smithi110.stdout:2343: seq_num 738 ranges {0=368640} 2024-09-14T03:45:43.741 INFO:tasks.rados.rados.0.smithi110.stdout:2343: writing smithi11075562-7 from 0 to 368640 tid 1 2024-09-14T03:45:43.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 235 2024-09-14T03:45:43.741 INFO:tasks.rados.rados.0.smithi110.stdout:2344: seq_num 739 ranges {0=385024} 2024-09-14T03:45:43.744 INFO:tasks.rados.rados.0.smithi110.stdout:2344: writing smithi11075562-37 from 0 to 385024 tid 1 2024-09-14T03:45:43.744 INFO:tasks.rados.rados.0.smithi110.stdout:2345: snap_create 2024-09-14T03:45:43.750 INFO:tasks.rados.rados.0.smithi110.stdout:2344: finishing write tid 1 to smithi11075562-37 2024-09-14T03:45:43.751 INFO:tasks.rados.rados.0.smithi110.stdout:2342: finishing rollback tid 0 to smithi11075562-44 2024-09-14T03:45:43.751 INFO:tasks.rados.rados.0.smithi110.stdout:2342: finishing rollback tid 1 to smithi11075562-44 2024-09-14T03:45:43.751 INFO:tasks.rados.rados.0.smithi110.stdout:2342: finishing rollback tid 2 to smithi11075562-44 2024-09-14T03:45:43.751 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 410 (ObjNum 712 snap 227 seq_num 712) dirty exists 2024-09-14T03:45:43.752 INFO:tasks.rados.rados.0.smithi110.stdout:2343: finishing write tid 1 to smithi11075562-7 2024-09-14T03:45:43.752 INFO:tasks.rados.rados.0.smithi110.stdout:2344: finishing write tid 2 to smithi11075562-37 2024-09-14T03:45:43.753 INFO:tasks.rados.rados.0.smithi110.stdout:2343: finishing write tid 2 to smithi11075562-7 2024-09-14T03:45:43.754 INFO:tasks.rados.rados.0.smithi110.stdout:2344: finishing write tid 3 to smithi11075562-37 2024-09-14T03:45:43.755 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1655 (ObjNum 739 snap 235 seq_num 739) dirty exists 2024-09-14T03:45:43.755 INFO:tasks.rados.rados.0.smithi110.stdout:2344: left oid 37 (ObjNum 739 snap 235 seq_num 739) 2024-09-14T03:45:43.755 INFO:tasks.rados.rados.0.smithi110.stdout:2343: finishing write tid 3 to smithi11075562-7 2024-09-14T03:45:43.755 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1572 (ObjNum 738 snap 235 seq_num 738) dirty exists 2024-09-14T03:45:43.755 INFO:tasks.rados.rados.0.smithi110.stdout:2343: left oid 7 (ObjNum 738 snap 235 seq_num 738) 2024-09-14T03:45:43.959 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:45:43 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:45:43.752+0000 7f0e0734f640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 121 after trimming was started 2024-09-14T03:45:43.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:43 smithi195 podman[100606]: 2024-09-14 03:45:43.771049386 +0000 UTC m=+0.548263212 container died 7b56745b654547d439d829f9bd388724adf53a06c52a45712c20cf97275d50dd (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-14T03:45:44.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:44 smithi195 podman[100629]: 2024-09-14 03:45:44.412296109 +0000 UTC m=+0.629451616 container remove 7b56745b654547d439d829f9bd388724adf53a06c52a45712c20cf97275d50dd (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-09-14T03:45:44.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:44 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-14T03:45:44.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:44 smithi195 systemd[1]: Stopped Ceph osd.7 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:45:44.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:44 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.7.service: Consumed 15.800s CPU time. 2024-09-14T03:45:44.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:44 smithi195 systemd[1]: Starting Ceph osd.7 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:2342: done (4 left) 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:2343: done (3 left) 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:2344: done (2 left) 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:2345: done (1 left) 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:2346: rollback oid 9 current snap is 236 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 9 to 234 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:2347: read oid 21 snap -1 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:2347: expect (ObjNum 696 snap 219 seq_num 696) 2024-09-14T03:45:44.745 INFO:tasks.rados.rados.0.smithi110.stdout:2348: snap_remove snap 217 2024-09-14T03:45:45.003 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:44 smithi195 ceph-mon[82388]: osdmap e542: 8 total, 7 up, 8 in 2024-09-14T03:45:45.003 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:44 smithi195 ceph-mon[82388]: pgmap v442: 121 pgs: 17 stale+active+clean, 9 peering, 95 active+clean; 219 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 120 KiB/s wr, 4 op/s 2024-09-14T03:45:45.003 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:44 smithi195 podman[100727]: 2024-09-14 03:45:44.757635191 +0000 UTC m=+0.022427825 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:45:45.004 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:44 smithi195 podman[100727]: 2024-09-14 03:45:44.855356462 +0000 UTC m=+0.120149094 container create 726d95c10f51ff058dc3e1a2dcf1096b5f0f9602aef989dcb039992c12029189 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T03:45:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:44 smithi110 ceph-mon[88262]: osdmap e542: 8 total, 7 up, 8 in 2024-09-14T03:45:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:44 smithi110 ceph-mon[88262]: pgmap v442: 121 pgs: 17 stale+active+clean, 9 peering, 95 active+clean; 219 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 120 KiB/s wr, 4 op/s 2024-09-14T03:45:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:44 smithi110 ceph-mon[86423]: osdmap e542: 8 total, 7 up, 8 in 2024-09-14T03:45:45.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:44 smithi110 ceph-mon[86423]: pgmap v442: 121 pgs: 17 stale+active+clean, 9 peering, 95 active+clean; 219 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 120 KiB/s wr, 4 op/s 2024-09-14T03:45:45.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 podman[100727]: 2024-09-14 03:45:45.108122656 +0000 UTC m=+0.372915280 container init 726d95c10f51ff058dc3e1a2dcf1096b5f0f9602aef989dcb039992c12029189 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-09-14T03:45:45.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 podman[100727]: 2024-09-14 03:45:45.111496468 +0000 UTC m=+0.376289097 container start 726d95c10f51ff058dc3e1a2dcf1096b5f0f9602aef989dcb039992c12029189 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T03:45:45.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 podman[100727]: 2024-09-14 03:45:45.216632352 +0000 UTC m=+0.481424978 container attach 726d95c10f51ff058dc3e1a2dcf1096b5f0f9602aef989dcb039992c12029189 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-14T03:45:45.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:45:45.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 bash[100727]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:45:45.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:45:45.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 bash[100727]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:45:45.748 INFO:tasks.rados.rados.0.smithi110.stdout:2346: finishing rollback tid 1 to smithi11075562-9 2024-09-14T03:45:45.749 INFO:tasks.rados.rados.0.smithi110.stdout:2346: finishing rollback tid 2 to smithi11075562-9 2024-09-14T03:45:45.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1565 (ObjNum 654 snap 202 seq_num 654) dirty exists 2024-09-14T03:45:45.749 INFO:tasks.rados.rados.0.smithi110.stdout:2347: expect (ObjNum 696 snap 219 seq_num 696) 2024-09-14T03:45:46.000 INFO:tasks.rados.rados.0.smithi110.stdout:2339: done (3 left) 2024-09-14T03:45:46.000 INFO:tasks.rados.rados.0.smithi110.stdout:2346: done (2 left) 2024-09-14T03:45:46.000 INFO:tasks.rados.rados.0.smithi110.stdout:2347: done (1 left) 2024-09-14T03:45:46.001 INFO:tasks.rados.rados.0.smithi110.stdout:2348: done (0 left) 2024-09-14T03:45:46.001 INFO:tasks.rados.rados.0.smithi110.stdout:2349: copy_from oid 50 from oid 39 current snap is 236 2024-09-14T03:45:46.001 INFO:tasks.rados.rados.0.smithi110.stdout:2350: snap_create 2024-09-14T03:45:46.002 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:45 smithi195 ceph-mon[82388]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T03:45:46.002 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:45 smithi195 ceph-mon[82388]: osdmap e543: 8 total, 7 up, 8 in 2024-09-14T03:45:46.002 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-14T03:45:46.002 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 bash[100727]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-14T03:45:46.002 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T03:45:46.002 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:45 smithi195 bash[100727]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T03:45:46.002 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T03:45:46.002 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 bash[100727]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T03:45:46.046 INFO:tasks.rados.rados.0.smithi110.stdout:2349: finishing copy_from racing read to smithi11075562-50 2024-09-14T03:45:46.049 INFO:tasks.rados.rados.0.smithi110.stdout:2349: finishing copy_from to smithi11075562-50 2024-09-14T03:45:46.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 1491 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:45:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:45 smithi110 ceph-mon[88262]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T03:45:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:45 smithi110 ceph-mon[88262]: osdmap e543: 8 total, 7 up, 8 in 2024-09-14T03:45:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:45 smithi110 ceph-mon[86423]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T03:45:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:45 smithi110 ceph-mon[86423]: osdmap e543: 8 total, 7 up, 8 in 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 bash[100727]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 bash[100727]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 bash[100727]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate[100741]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 bash[100727]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-14T03:45:46.321 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 podman[100727]: 2024-09-14 03:45:46.031044854 +0000 UTC m=+1.295837484 container died 726d95c10f51ff058dc3e1a2dcf1096b5f0f9602aef989dcb039992c12029189 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-14T03:45:46.746 INFO:tasks.rados.rados.0.smithi110.stdout:2349: done (1 left) 2024-09-14T03:45:46.746 INFO:tasks.rados.rados.0.smithi110.stdout:2350: done (0 left) 2024-09-14T03:45:46.746 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 237 2024-09-14T03:45:46.746 INFO:tasks.rados.rados.0.smithi110.stdout:2351: seq_num 740 ranges {2015232=401408} 2024-09-14T03:45:46.759 INFO:tasks.rados.rados.0.smithi110.stdout:2351: writing smithi11075562-40 from 2015232 to 2416640 tid 1 2024-09-14T03:45:46.760 INFO:tasks.rados.rados.0.smithi110.stdout:2352: copy_from oid 27 from oid 9 current snap is 237 2024-09-14T03:45:46.760 INFO:tasks.rados.rados.0.smithi110.stdout:2353: snap_remove snap 234 2024-09-14T03:45:46.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:46 smithi195 ceph-mon[82388]: osdmap e544: 8 total, 7 up, 8 in 2024-09-14T03:45:46.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:46 smithi195 ceph-mon[82388]: pgmap v445: 121 pgs: 4 active+clean+remapped, 21 active+undersized, 2 remapped+peering, 5 stale+active+clean, 9 peering, 10 active+undersized+degraded, 70 active+clean; 224 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 1.3 MiB/s wr, 12 op/s; 35/399 objects degraded (8.772%); 4/399 objects misplaced (1.003%) 2024-09-14T03:45:46.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:46 smithi195 ceph-mon[82388]: osdmap e545: 8 total, 7 up, 8 in 2024-09-14T03:45:46.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 podman[100881]: 2024-09-14 03:45:46.637012256 +0000 UTC m=+0.594901165 container remove 726d95c10f51ff058dc3e1a2dcf1096b5f0f9602aef989dcb039992c12029189 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T03:45:46.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 podman[100942]: 2024-09-14 03:45:46.751965531 +0000 UTC m=+0.018756191 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:45:46.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:46 smithi195 podman[100942]: 2024-09-14 03:45:46.871425473 +0000 UTC m=+0.138216176 container create 98e6e5252ce4da1fbeae53845b24bd2038c1f25dc360b89413492a75a7d3ba1e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-14T03:45:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:46 smithi110 ceph-mon[86423]: osdmap e544: 8 total, 7 up, 8 in 2024-09-14T03:45:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:46 smithi110 ceph-mon[86423]: pgmap v445: 121 pgs: 4 active+clean+remapped, 21 active+undersized, 2 remapped+peering, 5 stale+active+clean, 9 peering, 10 active+undersized+degraded, 70 active+clean; 224 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 1.3 MiB/s wr, 12 op/s; 35/399 objects degraded (8.772%); 4/399 objects misplaced (1.003%) 2024-09-14T03:45:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:46 smithi110 ceph-mon[86423]: osdmap e545: 8 total, 7 up, 8 in 2024-09-14T03:45:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:46 smithi110 ceph-mon[88262]: osdmap e544: 8 total, 7 up, 8 in 2024-09-14T03:45:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:46 smithi110 ceph-mon[88262]: pgmap v445: 121 pgs: 4 active+clean+remapped, 21 active+undersized, 2 remapped+peering, 5 stale+active+clean, 9 peering, 10 active+undersized+degraded, 70 active+clean; 224 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 1.3 MiB/s wr, 12 op/s; 35/399 objects degraded (8.772%); 4/399 objects misplaced (1.003%) 2024-09-14T03:45:47.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:46 smithi110 ceph-mon[88262]: osdmap e545: 8 total, 7 up, 8 in 2024-09-14T03:45:47.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:47 smithi195 podman[100942]: 2024-09-14 03:45:47.09862064 +0000 UTC m=+0.365411300 container init 98e6e5252ce4da1fbeae53845b24bd2038c1f25dc360b89413492a75a7d3ba1e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T03:45:47.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:47 smithi195 podman[100942]: 2024-09-14 03:45:47.10180679 +0000 UTC m=+0.368597448 container start 98e6e5252ce4da1fbeae53845b24bd2038c1f25dc360b89413492a75a7d3ba1e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-14T03:45:47.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:47 smithi195 ceph-osd[100961]: -- 172.21.15.195:0/707653665 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556965b99a40 con 0x556965b77c00 2024-09-14T03:45:47.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:47 smithi195 bash[100942]: 98e6e5252ce4da1fbeae53845b24bd2038c1f25dc360b89413492a75a7d3ba1e 2024-09-14T03:45:47.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:47 smithi195 systemd[1]: Started Ceph osd.7 for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:45:47.762 INFO:tasks.rados.rados.0.smithi110.stdout:2353: done (2 left) 2024-09-14T03:45:47.762 INFO:tasks.rados.rados.0.smithi110.stdout:2354: copy_from oid 32 from oid 2 current snap is 237 2024-09-14T03:45:47.762 INFO:tasks.rados.rados.0.smithi110.stdout:2351: finishing write tid 1 to smithi11075562-40 2024-09-14T03:45:47.762 INFO:tasks.rados.rados.0.smithi110.stdout:2351: finishing write tid 2 to smithi11075562-40 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2351: finishing write tid 3 to smithi11075562-40 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1251 (ObjNum 740 snap 237 seq_num 740) dirty exists 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2351: left oid 40 (ObjNum 740 snap 237 seq_num 740) 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2352: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2352: finishing copy_from to smithi11075562-27 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1565 (ObjNum 654 snap 202 seq_num 654) dirty exists 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2351: done (2 left) 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2352: done (1 left) 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2355: rmattr oid 44 current snap is 237 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2355: done (1 left) 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2356: copy_from oid 6 from oid 25 current snap is 237 2024-09-14T03:45:47.763 INFO:tasks.rados.rados.0.smithi110.stdout:2357: snap_create 2024-09-14T03:45:47.768 INFO:tasks.rados.rados.0.smithi110.stdout:2354: finishing copy_from to smithi11075562-32 2024-09-14T03:45:47.768 INFO:tasks.rados.rados.0.smithi110.stdout:2354: got expected ENOENT (src dne) 2024-09-14T03:45:47.768 INFO:tasks.rados.rados.0.smithi110.stdout:2354: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:45:47.794 INFO:tasks.rados.rados.0.smithi110.stdout:2356: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:45:47.798 INFO:tasks.rados.rados.0.smithi110.stdout:2356: finishing copy_from to smithi11075562-6 2024-09-14T03:45:47.798 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1443 (ObjNum 642 snap 199 seq_num 642) dirty exists 2024-09-14T03:45:48.128 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:47 smithi195 ceph-mon[82388]: Health check failed: Degraded data redundancy: 35/399 objects degraded (8.772%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:48.128 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:47 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:48.128 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:47 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:48.128 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:47 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:47 smithi110 ceph-mon[86423]: Health check failed: Degraded data redundancy: 35/399 objects degraded (8.772%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:47 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:47 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:47 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:47 smithi110 ceph-mon[88262]: Health check failed: Degraded data redundancy: 35/399 objects degraded (8.772%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:47 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:48.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:47 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:48.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:47 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:48.740 INFO:tasks.rados.rados.0.smithi110.stdout:2354: done (2 left) 2024-09-14T03:45:48.741 INFO:tasks.rados.rados.0.smithi110.stdout:2356: done (1 left) 2024-09-14T03:45:48.741 INFO:tasks.rados.rados.0.smithi110.stdout:2357: done (0 left) 2024-09-14T03:45:48.741 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 238 2024-09-14T03:45:48.741 INFO:tasks.rados.rados.0.smithi110.stdout:2358: seq_num 741 ranges {2416640=417792} 2024-09-14T03:45:48.755 INFO:tasks.rados.rados.0.smithi110.stdout:2358: writing smithi11075562-40 from 2416640 to 2834432 tid 1 2024-09-14T03:45:48.755 INFO:tasks.rados.rados.0.smithi110.stdout:2359: setattr oid 23 current snap is 238 2024-09-14T03:45:48.756 INFO:tasks.rados.rados.0.smithi110.stdout:2360: delete oid 21 current snap is 238 2024-09-14T03:45:48.761 INFO:tasks.rados.rados.0.smithi110.stdout:2358: finishing write tid 1 to smithi11075562-40 2024-09-14T03:45:48.762 INFO:tasks.rados.rados.0.smithi110.stdout:2358: finishing write tid 2 to smithi11075562-40 2024-09-14T03:45:48.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1484 (ObjNum 724 snap 229 seq_num 724) dirty exists 2024-09-14T03:45:48.763 INFO:tasks.rados.rados.0.smithi110.stdout:2358: finishing write tid 3 to smithi11075562-40 2024-09-14T03:45:48.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1445 (ObjNum 741 snap 238 seq_num 741) dirty exists 2024-09-14T03:45:48.763 INFO:tasks.rados.rados.0.smithi110.stdout:2358: left oid 40 (ObjNum 741 snap 238 seq_num 741) 2024-09-14T03:45:48.764 INFO:tasks.rados.rados.0.smithi110.stdout:2358: done (2 left) 2024-09-14T03:45:48.764 INFO:tasks.rados.rados.0.smithi110.stdout:2359: done (1 left) 2024-09-14T03:45:48.764 INFO:tasks.rados.rados.0.smithi110.stdout:2360: done (0 left) 2024-09-14T03:45:48.764 INFO:tasks.rados.rados.0.smithi110.stdout:2361: read oid 41 snap -1 2024-09-14T03:45:48.764 INFO:tasks.rados.rados.0.smithi110.stdout:2361: expect (ObjNum 734 snap 233 seq_num 734) 2024-09-14T03:45:48.764 INFO:tasks.rados.rados.0.smithi110.stdout:2362: copy_from oid 38 from oid 8 current snap is 238 2024-09-14T03:45:48.764 INFO:tasks.rados.rados.0.smithi110.stdout:2363: copy_from oid 42 from oid 40 current snap is 238 2024-09-14T03:45:48.764 INFO:tasks.rados.rados.0.smithi110.stdout:2364: snap_remove snap 237 2024-09-14T03:45:49.064 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:45:48.742+0000 7f1a6d35a640 -1 Falling back to public interface 2024-09-14T03:45:49.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:48 smithi195 ceph-mon[82388]: osdmap e546: 8 total, 7 up, 8 in 2024-09-14T03:45:49.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:48 smithi195 ceph-mon[82388]: pgmap v448: 121 pgs: 1 active+clean+snaptrim, 6 active+clean+remapped, 35 active+undersized, 2 remapped+peering, 13 active+undersized+degraded, 64 active+clean; 225 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 11 op/s; 46/396 objects degraded (11.616%); 6/396 objects misplaced (1.515%); 6.8 MiB/s, 25 objects/s recovering 2024-09-14T03:45:49.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:48 smithi195 ceph-mon[82388]: osdmap e547: 8 total, 7 up, 8 in 2024-09-14T03:45:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:48 smithi110 ceph-mon[88262]: osdmap e546: 8 total, 7 up, 8 in 2024-09-14T03:45:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:48 smithi110 ceph-mon[88262]: pgmap v448: 121 pgs: 1 active+clean+snaptrim, 6 active+clean+remapped, 35 active+undersized, 2 remapped+peering, 13 active+undersized+degraded, 64 active+clean; 225 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 11 op/s; 46/396 objects degraded (11.616%); 6/396 objects misplaced (1.515%); 6.8 MiB/s, 25 objects/s recovering 2024-09-14T03:45:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:48 smithi110 ceph-mon[88262]: osdmap e547: 8 total, 7 up, 8 in 2024-09-14T03:45:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:48 smithi110 ceph-mon[86423]: osdmap e546: 8 total, 7 up, 8 in 2024-09-14T03:45:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:48 smithi110 ceph-mon[86423]: pgmap v448: 121 pgs: 1 active+clean+snaptrim, 6 active+clean+remapped, 35 active+undersized, 2 remapped+peering, 13 active+undersized+degraded, 64 active+clean; 225 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 11 op/s; 46/396 objects degraded (11.616%); 6/396 objects misplaced (1.515%); 6.8 MiB/s, 25 objects/s recovering 2024-09-14T03:45:49.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:48 smithi110 ceph-mon[86423]: osdmap e547: 8 total, 7 up, 8 in 2024-09-14T03:45:49.767 INFO:tasks.rados.rados.0.smithi110.stdout:2362: finishing copy_from to smithi11075562-38 2024-09-14T03:45:49.768 INFO:tasks.rados.rados.0.smithi110.stdout:2362: got expected ENOENT (src dne) 2024-09-14T03:45:49.768 INFO:tasks.rados.rados.0.smithi110.stdout:2362: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:45:49.768 INFO:tasks.rados.rados.0.smithi110.stdout:2361: expect (ObjNum 734 snap 233 seq_num 734) 2024-09-14T03:45:49.780 INFO:tasks.rados.rados.0.smithi110.stdout:2363: finishing copy_from racing read to smithi11075562-42 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2363: finishing copy_from to smithi11075562-42 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1445 (ObjNum 741 snap 238 seq_num 741) dirty exists 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2361: done (3 left) 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2362: done (2 left) 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2363: done (1 left) 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2364: done (0 left) 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2365: rollback oid 17 current snap is 238 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 214 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2366: rollback oid 9 current snap is 238 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 9 to 235 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2367: read oid 48 snap -1 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2367: expect deleted 2024-09-14T03:45:49.781 INFO:tasks.rados.rados.0.smithi110.stdout:2368: rollback oid 47 current snap is 238 2024-09-14T03:45:49.782 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 47 to 233 2024-09-14T03:45:49.782 INFO:tasks.rados.rados.0.smithi110.stdout:2369: snap_remove snap 226 2024-09-14T03:45:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:45:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:45:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:2366: finishing rollback tid 0 to smithi11075562-9 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:2366: finishing rollback tid 1 to smithi11075562-9 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1567 (ObjNum 129 snap 0 seq_num 66784248) dirty dne 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:2365: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:2365: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:2365: finishing rollback tid 2 to smithi11075562-17 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1139 (ObjNum 668 snap 206 seq_num 668) dirty exists 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:2368: finishing rollback tid 0 to smithi11075562-47 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:2368: finishing rollback tid 1 to smithi11075562-47 2024-09-14T03:45:50.773 INFO:tasks.rados.rados.0.smithi110.stdout:2368: finishing rollback tid 2 to smithi11075562-47 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1669 (ObjNum 674 snap 209 seq_num 674) dirty exists 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:2365: done (4 left) 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:2366: done (3 left) 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:2367: done (2 left) 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:2368: done (1 left) 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:2369: done (0 left) 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:2370: rollback oid 10 current snap is 238 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 232 2024-09-14T03:45:50.774 INFO:tasks.rados.rados.0.smithi110.stdout:2371: delete oid 37 current snap is 238 2024-09-14T03:45:50.780 INFO:tasks.rados.rados.0.smithi110.stdout:2371: done (1 left) 2024-09-14T03:45:50.781 INFO:tasks.rados.rados.0.smithi110.stdout:2372: rollback oid 35 current snap is 238 2024-09-14T03:45:50.781 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 229 2024-09-14T03:45:50.781 INFO:tasks.rados.rados.0.smithi110.stdout:2373: snap_create 2024-09-14T03:45:50.784 INFO:tasks.rados.rados.0.smithi110.stdout:2370: finishing rollback tid 0 to smithi11075562-10 2024-09-14T03:45:50.784 INFO:tasks.rados.rados.0.smithi110.stdout:2370: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:45:50.784 INFO:tasks.rados.rados.0.smithi110.stdout:2370: finishing rollback tid 2 to smithi11075562-10 2024-09-14T03:45:50.785 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1487 (ObjNum 666 snap 205 seq_num 666) dirty exists 2024-09-14T03:45:50.789 INFO:tasks.rados.rados.0.smithi110.stdout:2372: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:45:50.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 0 (ObjNum 65 snap 0 seq_num 1533759079) dirty dne 2024-09-14T03:45:51.029 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:50 smithi195 ceph-mon[82388]: osdmap e548: 8 total, 7 up, 8 in 2024-09-14T03:45:51.029 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:50 smithi195 ceph-mon[82388]: pgmap v451: 121 pgs: 1 active+clean+snaptrim, 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 64 active+clean; 241 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.8 MiB/s wr, 8 op/s; 46/399 objects degraded (11.529%); 7/399 objects misplaced (1.754%); 6.8 MiB/s, 25 objects/s recovering 2024-09-14T03:45:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:50 smithi110 ceph-mon[86423]: osdmap e548: 8 total, 7 up, 8 in 2024-09-14T03:45:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:50 smithi110 ceph-mon[86423]: pgmap v451: 121 pgs: 1 active+clean+snaptrim, 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 64 active+clean; 241 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.8 MiB/s wr, 8 op/s; 46/399 objects degraded (11.529%); 7/399 objects misplaced (1.754%); 6.8 MiB/s, 25 objects/s recovering 2024-09-14T03:45:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:50 smithi110 ceph-mon[88262]: osdmap e548: 8 total, 7 up, 8 in 2024-09-14T03:45:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:50 smithi110 ceph-mon[88262]: pgmap v451: 121 pgs: 1 active+clean+snaptrim, 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 64 active+clean; 241 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.8 MiB/s wr, 8 op/s; 46/399 objects degraded (11.529%); 7/399 objects misplaced (1.754%); 6.8 MiB/s, 25 objects/s recovering 2024-09-14T03:45:51.777 INFO:tasks.rados.rados.0.smithi110.stdout:2370: done (2 left) 2024-09-14T03:45:51.777 INFO:tasks.rados.rados.0.smithi110.stdout:2372: done (1 left) 2024-09-14T03:45:51.777 INFO:tasks.rados.rados.0.smithi110.stdout:2373: done (0 left) 2024-09-14T03:45:51.777 INFO:tasks.rados.rados.0.smithi110.stdout:2374: snap_create 2024-09-14T03:45:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:45:51] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.51.0" 2024-09-14T03:45:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-mon[88262]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T03:45:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-mon[88262]: osdmap e549: 8 total, 7 up, 8 in 2024-09-14T03:45:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:51.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-mon[86423]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T03:45:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-mon[86423]: osdmap e549: 8 total, 7 up, 8 in 2024-09-14T03:45:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:51 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:51 smithi195 ceph-mon[82388]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T03:45:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:51 smithi195 ceph-mon[82388]: osdmap e549: 8 total, 7 up, 8 in 2024-09-14T03:45:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:51 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:51 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:52.554 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:45:52.488+0000 7f1a6d35a640 -1 osd.7 539 log_to_monitors true 2024-09-14T03:45:52.785 INFO:tasks.rados.rados.0.smithi110.stdout:2374: done (0 left) 2024-09-14T03:45:52.785 INFO:tasks.rados.rados.0.smithi110.stdout:2375: snap_create 2024-09-14T03:45:52.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:52 smithi195 ceph-mon[82388]: osdmap e550: 8 total, 7 up, 8 in 2024-09-14T03:45:52.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:52 smithi195 ceph-mon[82388]: pgmap v454: 121 pgs: 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 65 active+clean; 252 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 6.6 MiB/s wr, 9 op/s; 49/411 objects degraded (11.922%); 7/411 objects misplaced (1.703%) 2024-09-14T03:45:52.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:52 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:52.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:52 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:52.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:52 smithi195 ceph-mon[82388]: from='osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:45:52.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:52 smithi195 ceph-mon[82388]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:45:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[88262]: osdmap e550: 8 total, 7 up, 8 in 2024-09-14T03:45:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[88262]: pgmap v454: 121 pgs: 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 65 active+clean; 252 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 6.6 MiB/s wr, 9 op/s; 49/411 objects degraded (11.922%); 7/411 objects misplaced (1.703%) 2024-09-14T03:45:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:53.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[88262]: from='osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:45:53.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[88262]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:45:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[86423]: osdmap e550: 8 total, 7 up, 8 in 2024-09-14T03:45:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[86423]: pgmap v454: 121 pgs: 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 65 active+clean; 252 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 6.6 MiB/s wr, 9 op/s; 49/411 objects degraded (11.922%); 7/411 objects misplaced (1.703%) 2024-09-14T03:45:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[86423]: from='osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:45:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:52 smithi110 ceph-mon[86423]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:45:53.339 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:45:53 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:45:53.072+0000 7f1a610fc640 -1 osd.7 539 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:45:53.789 INFO:tasks.rados.rados.0.smithi110.stdout:2375: done (0 left) 2024-09-14T03:45:53.789 INFO:tasks.rados.rados.0.smithi110.stdout:2376: read oid 21 snap -1 2024-09-14T03:45:53.789 INFO:tasks.rados.rados.0.smithi110.stdout:2376: expect deleted 2024-09-14T03:45:53.789 INFO:tasks.rados.rados.0.smithi110.stdout:2377: delete oid 10 current snap is 241 2024-09-14T03:45:53.883 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:53 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 49/411 objects degraded (11.922%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:53.883 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:53 smithi195 ceph-mon[82388]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:45:53.883 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:53 smithi195 ceph-mon[82388]: from='osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:53.884 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:53 smithi195 ceph-mon[82388]: osdmap e551: 8 total, 7 up, 8 in 2024-09-14T03:45:53.884 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:53 smithi195 ceph-mon[82388]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:53.884 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:53 smithi195 ceph-mon[82388]: from='osd.7 ' entity='osd.7' 2024-09-14T03:45:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 49/411 objects degraded (11.922%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[88262]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:45:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[88262]: from='osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[88262]: osdmap e551: 8 total, 7 up, 8 in 2024-09-14T03:45:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[88262]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:54.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[88262]: from='osd.7 ' entity='osd.7' 2024-09-14T03:45:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 49/411 objects degraded (11.922%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[86423]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:45:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[86423]: from='osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[86423]: osdmap e551: 8 total, 7 up, 8 in 2024-09-14T03:45:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[86423]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-14T03:45:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:53 smithi110 ceph-mon[86423]: from='osd.7 ' entity='osd.7' 2024-09-14T03:45:55.100 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:54 smithi195 ceph-mon[82388]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:45:55.100 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:54 smithi195 ceph-mon[82388]: osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926] boot 2024-09-14T03:45:55.100 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:54 smithi195 ceph-mon[82388]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T03:45:55.100 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:54 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:45:55.100 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:54 smithi195 ceph-mon[82388]: pgmap v457: 121 pgs: 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 65 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 2.9 MiB/s wr, 4 op/s; 49/405 objects degraded (12.099%); 7/405 objects misplaced (1.728%) 2024-09-14T03:45:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[88262]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:45:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[88262]: osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926] boot 2024-09-14T03:45:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[88262]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T03:45:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:45:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[88262]: pgmap v457: 121 pgs: 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 65 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 2.9 MiB/s wr, 4 op/s; 49/405 objects degraded (12.099%); 7/405 objects misplaced (1.728%) 2024-09-14T03:45:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[86423]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:45:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[86423]: osd.7 [v2:172.21.15.195:6824/72177926,v1:172.21.15.195:6825/72177926] boot 2024-09-14T03:45:55.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[86423]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T03:45:55.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:45:55.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:54 smithi110 ceph-mon[86423]: pgmap v457: 121 pgs: 7 active+clean+remapped, 1 active+clean+remapped+snaptrim, 35 active+undersized, 13 active+undersized+degraded, 65 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 2.9 MiB/s wr, 4 op/s; 49/405 objects degraded (12.099%); 7/405 objects misplaced (1.728%) 2024-09-14T03:45:55.818 INFO:tasks.rados.rados.0.smithi110.stdout:2376: done (1 left) 2024-09-14T03:45:55.818 INFO:tasks.rados.rados.0.smithi110.stdout:2377: done (0 left) 2024-09-14T03:45:55.818 INFO:tasks.rados.rados.0.smithi110.stdout:2378: rollback oid 38 current snap is 241 2024-09-14T03:45:55.818 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 214 2024-09-14T03:45:55.818 INFO:tasks.rados.rados.0.smithi110.stdout:2379: snap_remove snap 232 2024-09-14T03:45:56.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T03:45:56.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:45:56.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:45:56.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-14T03:45:56.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:45:56.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:45:56.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-14T03:45:56.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:45:56.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:45:56.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-14T03:45:56.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:45:56.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:45:56.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-14T03:45:56.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:45:56.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:45:56.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-14T03:45:56.159 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T03:45:56.159 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:45:56.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:45:56.162 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[88262]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-14T03:45:56.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:45:56.164 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:45:56.164 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-14T03:45:56.164 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-14T03:45:56.164 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:55 smithi110 ceph-mon[86423]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T03:45:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T03:45:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:45:56.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:45:56.211 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-14T03:45:56.212 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:55 smithi195 ceph-mon[82388]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T03:45:56.798 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all osd 2024-09-14T03:45:56.805 INFO:tasks.rados.rados.0.smithi110.stdout:2379: done (1 left) 2024-09-14T03:45:56.805 INFO:tasks.rados.rados.0.smithi110.stdout:2380: snap_create 2024-09-14T03:45:56.805 INFO:tasks.rados.rados.0.smithi110.stdout:2378: finishing rollback tid 0 to smithi11075562-38 2024-09-14T03:45:56.806 INFO:tasks.rados.rados.0.smithi110.stdout:2378: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:45:56.806 INFO:tasks.rados.rados.0.smithi110.stdout:2378: finishing rollback tid 2 to smithi11075562-38 2024-09-14T03:45:56.806 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1575 (ObjNum 664 snap 204 seq_num 664) dirty exists 2024-09-14T03:45:57.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[86423]: Upgrade: Setting require_osd_release to 19 squid 2024-09-14T03:45:57.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[86423]: pgmap v459: 121 pgs: 12 peering, 6 remapped+peering, 2 active+clean+remapped, 21 active+undersized, 8 active+undersized+degraded, 72 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 226 KiB/s rd, 0 B/s wr, 2 op/s; 34/402 objects degraded (8.458%); 2/402 objects misplaced (0.498%) 2024-09-14T03:45:57.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-14T03:45:57.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[86423]: Upgrade: It appears safe to stop mds.a 2024-09-14T03:45:57.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all osd 2024-09-14T03:45:57.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[88262]: Upgrade: Setting require_osd_release to 19 squid 2024-09-14T03:45:57.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[88262]: pgmap v459: 121 pgs: 12 peering, 6 remapped+peering, 2 active+clean+remapped, 21 active+undersized, 8 active+undersized+degraded, 72 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 226 KiB/s rd, 0 B/s wr, 2 op/s; 34/402 objects degraded (8.458%); 2/402 objects misplaced (0.498%) 2024-09-14T03:45:57.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-14T03:45:57.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:56 smithi110 ceph-mon[88262]: Upgrade: It appears safe to stop mds.a 2024-09-14T03:45:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:56 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all osd 2024-09-14T03:45:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:56 smithi195 ceph-mon[82388]: Upgrade: Setting require_osd_release to 19 squid 2024-09-14T03:45:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:56 smithi195 ceph-mon[82388]: pgmap v459: 121 pgs: 12 peering, 6 remapped+peering, 2 active+clean+remapped, 21 active+undersized, 8 active+undersized+degraded, 72 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 226 KiB/s rd, 0 B/s wr, 2 op/s; 34/402 objects degraded (8.458%); 2/402 objects misplaced (0.498%) 2024-09-14T03:45:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:56 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-14T03:45:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:56 smithi195 ceph-mon[82388]: Upgrade: It appears safe to stop mds.a 2024-09-14T03:45:57.806 INFO:tasks.rados.rados.0.smithi110.stdout:2378: done (1 left) 2024-09-14T03:45:57.806 INFO:tasks.rados.rados.0.smithi110.stdout:2380: done (0 left) 2024-09-14T03:45:57.806 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 242 2024-09-14T03:45:57.806 INFO:tasks.rados.rados.0.smithi110.stdout:2381: seq_num 745 ranges {2834432=483328} 2024-09-14T03:45:57.824 INFO:tasks.rados.rados.0.smithi110.stdout:2381: writing smithi11075562-40 from 2834432 to 3317760 tid 1 2024-09-14T03:45:57.825 INFO:tasks.rados.rados.0.smithi110.stdout:2382: setattr oid 18 current snap is 242 2024-09-14T03:45:57.826 INFO:tasks.rados.rados.0.smithi110.stdout:2383: setattr oid 7 current snap is 242 2024-09-14T03:45:57.828 INFO:tasks.rados.rados.0.smithi110.stdout:2381: finishing write tid 1 to smithi11075562-40 2024-09-14T03:45:57.829 INFO:tasks.rados.rados.0.smithi110.stdout:2384: rollback oid 12 current snap is 242 2024-09-14T03:45:57.829 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 238 2024-09-14T03:45:57.829 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 242 2024-09-14T03:45:57.829 INFO:tasks.rados.rados.0.smithi110.stdout:2385: seq_num 746 ranges {294912=499712} 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2385: writing smithi11075562-4 from 294912 to 794624 tid 1 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2386: read oid 32 snap 229 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2386: expect (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2387: copy_from oid 46 from oid 20 current snap is 242 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2384: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 0 (ObjNum 129 snap 0 seq_num 64858640) dirty dne 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2381: finishing write tid 2 to smithi11075562-40 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2381: finishing write tid 3 to smithi11075562-40 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1447 (ObjNum 745 snap 242 seq_num 745) dirty exists 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2381: left oid 40 (ObjNum 745 snap 242 seq_num 745) 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1448 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:45:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2381: done (6 left) 2024-09-14T03:45:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2382: done (5 left) 2024-09-14T03:45:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2384: done (4 left) 2024-09-14T03:45:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2388: read oid 28 snap 238 2024-09-14T03:45:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2388: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:45:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 242 2024-09-14T03:45:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1576 (ObjNum 738 snap 235 seq_num 738) dirty exists 2024-09-14T03:45:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2389: seq_num 747 ranges {8306688=516096} 2024-09-14T03:45:57.882 INFO:tasks.rados.rados.0.smithi110.stdout:2389: writing smithi11075562-6 from 8306688 to 8822784 tid 1 2024-09-14T03:45:57.882 INFO:tasks.rados.rados.0.smithi110.stdout:2383: done (5 left) 2024-09-14T03:45:57.882 INFO:tasks.rados.rados.0.smithi110.stdout:2390: delete oid 41 current snap is 242 2024-09-14T03:45:57.883 INFO:tasks.rados.rados.0.smithi110.stdout:2388: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:45:57.896 INFO:tasks.rados.rados.0.smithi110.stdout:2385: finishing write tid 1 to smithi11075562-4 2024-09-14T03:45:57.897 INFO:tasks.rados.rados.0.smithi110.stdout:2385: finishing write tid 2 to smithi11075562-4 2024-09-14T03:45:57.897 INFO:tasks.rados.rados.0.smithi110.stdout:2385: finishing write tid 3 to smithi11075562-4 2024-09-14T03:45:57.897 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1222 (ObjNum 746 snap 242 seq_num 746) dirty exists 2024-09-14T03:45:57.897 INFO:tasks.rados.rados.0.smithi110.stdout:2385: left oid 4 (ObjNum 746 snap 242 seq_num 746) 2024-09-14T03:45:57.897 INFO:tasks.rados.rados.0.smithi110.stdout:2386: expect (ObjNum 572 snap 174 seq_num 572) 2024-09-14T03:45:57.958 INFO:tasks.rados.rados.0.smithi110.stdout:2387: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:45:57.958 INFO:tasks.rados.rados.0.smithi110.stdout:2387: finishing copy_from to smithi11075562-46 2024-09-14T03:45:57.958 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1562 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:45:57.958 INFO:tasks.rados.rados.0.smithi110.stdout:2389: finishing write tid 1 to smithi11075562-6 2024-09-14T03:45:57.958 INFO:tasks.rados.rados.0.smithi110.stdout:2389: finishing write tid 2 to smithi11075562-6 2024-09-14T03:45:57.958 INFO:tasks.rados.rados.0.smithi110.stdout:2389: finishing write tid 3 to smithi11075562-6 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1450 (ObjNum 747 snap 242 seq_num 747) dirty exists 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2389: left oid 6 (ObjNum 747 snap 242 seq_num 747) 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2385: done (5 left) 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2386: done (4 left) 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2387: done (3 left) 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2388: done (2 left) 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2389: done (1 left) 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2390: done (0 left) 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2391: rmattr oid 23 current snap is 242 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2392: read oid 49 snap 239 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2392: expect (ObjNum 723 snap 229 seq_num 723) 2024-09-14T03:45:57.959 INFO:tasks.rados.rados.0.smithi110.stdout:2393: snap_create 2024-09-14T03:45:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1489 (ObjNum 724 snap 229 seq_num 724) dirty exists 2024-09-14T03:45:57.969 INFO:tasks.rados.rados.0.smithi110.stdout:2392: expect (ObjNum 723 snap 229 seq_num 723) 2024-09-14T03:45:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T03:45:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: Upgrade: Updating mds.a 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: Deploying daemon mds.a on smithi110 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[86423]: pgmap v462: 121 pgs: 12 peering, 6 remapped+peering, 1 active+clean+remapped, 13 active+undersized, 6 active+undersized+degraded, 83 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 21/399 objects degraded (5.263%); 2/399 objects misplaced (0.501%); 3.8 MiB/s, 1 objects/s recovering 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: Upgrade: Updating mds.a 2024-09-14T03:45:58.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:58.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:45:58.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:45:58.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:58.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: Deploying daemon mds.a on smithi110 2024-09-14T03:45:58.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:57 smithi110 ceph-mon[88262]: pgmap v462: 121 pgs: 12 peering, 6 remapped+peering, 1 active+clean+remapped, 13 active+undersized, 6 active+undersized+degraded, 83 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 21/399 objects degraded (5.263%); 2/399 objects misplaced (0.501%); 3.8 MiB/s, 1 objects/s recovering 2024-09-14T03:45:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T03:45:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:45:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: Upgrade: Updating mds.a 2024-09-14T03:45:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:45:58.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:45:58.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:45:58.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:45:58.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: Deploying daemon mds.a on smithi110 2024-09-14T03:45:58.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:57 smithi195 ceph-mon[82388]: pgmap v462: 121 pgs: 12 peering, 6 remapped+peering, 1 active+clean+remapped, 13 active+undersized, 6 active+undersized+degraded, 83 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 21/399 objects degraded (5.263%); 2/399 objects misplaced (0.501%); 3.8 MiB/s, 1 objects/s recovering 2024-09-14T03:45:58.809 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:45:58 smithi110 systemd[1]: Stopping Ceph mds.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:45:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:2391: done (2 left) 2024-09-14T03:45:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:2392: done (1 left) 2024-09-14T03:45:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:2393: done (0 left) 2024-09-14T03:45:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 243 2024-09-14T03:45:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:2394: seq_num 749 ranges {1359872=548864} 2024-09-14T03:45:58.826 INFO:tasks.rados.rados.0.smithi110.stdout:2394: writing smithi11075562-27 from 1359872 to 1908736 tid 1 2024-09-14T03:45:58.826 INFO:tasks.rados.rados.0.smithi110.stdout:2395: setattr oid 31 current snap is 243 2024-09-14T03:45:58.828 INFO:tasks.rados.rados.0.smithi110.stdout:2396: read oid 26 snap -1 2024-09-14T03:45:58.828 INFO:tasks.rados.rados.0.smithi110.stdout:2396: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:45:58.828 INFO:tasks.rados.rados.0.smithi110.stdout:2397: read oid 12 snap -1 2024-09-14T03:45:58.828 INFO:tasks.rados.rados.0.smithi110.stdout:2397: expect deleted 2024-09-14T03:45:58.828 INFO:tasks.rados.rados.0.smithi110.stdout:2398: copy_from oid 1 from oid 33 current snap is 243 2024-09-14T03:45:58.828 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 243 2024-09-14T03:45:58.828 INFO:tasks.rados.rados.0.smithi110.stdout:2399: seq_num 750 ranges {4587520=565248} 2024-09-14T03:45:58.857 INFO:tasks.rados.rados.0.smithi110.stdout:2399: writing smithi11075562-5 from 4587520 to 5152768 tid 1 2024-09-14T03:45:58.857 INFO:tasks.rados.rados.0.smithi110.stdout:2400: snap_create 2024-09-14T03:45:58.857 INFO:tasks.rados.rados.0.smithi110.stdout:2396: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:45:58.860 INFO:tasks.rados.rados.0.smithi110.stdout:2394: finishing write tid 1 to smithi11075562-27 2024-09-14T03:45:58.860 INFO:tasks.rados.rados.0.smithi110.stdout:2394: finishing write tid 2 to smithi11075562-27 2024-09-14T03:45:58.860 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1223 (ObjNum 733 snap 233 seq_num 733) dirty exists 2024-09-14T03:45:58.860 INFO:tasks.rados.rados.0.smithi110.stdout:2394: finishing write tid 3 to smithi11075562-27 2024-09-14T03:45:58.860 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1567 (ObjNum 749 snap 243 seq_num 749) dirty exists 2024-09-14T03:45:58.860 INFO:tasks.rados.rados.0.smithi110.stdout:2394: left oid 27 (ObjNum 749 snap 243 seq_num 749) 2024-09-14T03:45:58.860 INFO:tasks.rados.rados.0.smithi110.stdout:2398: finishing copy_from racing read to smithi11075562-1 2024-09-14T03:45:58.861 INFO:tasks.rados.rados.0.smithi110.stdout:2398: finishing copy_from to smithi11075562-1 2024-09-14T03:45:58.861 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1465 (ObjNum 701 snap 224 seq_num 701) dirty exists 2024-09-14T03:45:58.862 INFO:tasks.rados.rados.0.smithi110.stdout:2399: finishing write tid 1 to smithi11075562-5 2024-09-14T03:45:58.862 INFO:tasks.rados.rados.0.smithi110.stdout:2399: finishing write tid 2 to smithi11075562-5 2024-09-14T03:45:58.864 INFO:tasks.rados.rados.0.smithi110.stdout:2399: finishing write tid 3 to smithi11075562-5 2024-09-14T03:45:58.864 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1688 (ObjNum 750 snap 243 seq_num 750) dirty exists 2024-09-14T03:45:58.864 INFO:tasks.rados.rados.0.smithi110.stdout:2399: left oid 5 (ObjNum 750 snap 243 seq_num 750) 2024-09-14T03:45:59.006 INFO:tasks.rados.rados.0.smithi110.stdout:2394: done (6 left) 2024-09-14T03:45:59.006 INFO:tasks.rados.rados.0.smithi110.stdout:2395: done (5 left) 2024-09-14T03:45:59.006 INFO:tasks.rados.rados.0.smithi110.stdout:2396: done (4 left) 2024-09-14T03:45:59.006 INFO:tasks.rados.rados.0.smithi110.stdout:2397: done (3 left) 2024-09-14T03:45:59.006 INFO:tasks.rados.rados.0.smithi110.stdout:2398: done (2 left) 2024-09-14T03:45:59.006 INFO:tasks.rados.rados.0.smithi110.stdout:2399: done (1 left) 2024-09-14T03:45:59.006 INFO:tasks.rados.rados.0.smithi110.stdout:2400: done (0 left) 2024-09-14T03:45:59.006 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 244 2024-09-14T03:45:59.007 INFO:tasks.rados.rados.0.smithi110.stdout:2401: seq_num 751 ranges {4710400=581632} 2024-09-14T03:45:59.035 INFO:tasks.rados.rados.0.smithi110.stdout:2401: writing smithi11075562-43 from 4710400 to 5292032 tid 1 2024-09-14T03:45:59.035 INFO:tasks.rados.rados.0.smithi110.stdout:2402: snap_remove snap 214 2024-09-14T03:45:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:58 smithi110 ceph-mon[88262]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T03:45:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:58 smithi110 ceph-mon[88262]: Health check update: Degraded data redundancy: 21/399 objects degraded (5.263%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:58 smithi110 ceph-mon[86423]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T03:45:59.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:58 smithi110 ceph-mon[86423]: Health check update: Degraded data redundancy: 21/399 objects degraded (5.263%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:45:59.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[86419]: 2024-09-14T03:45:58.999+0000 7fa26c043640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:45:59.197 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:45:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[61551]: 2024-09-14T03:45:58.998+0000 7fb8f4433640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:45:59.197 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:45:58 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[61551]: 2024-09-14T03:45:58.998+0000 7fb8f4433640 -1 mds.a *** got signal Terminated *** 2024-09-14T03:45:59.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:58 smithi195 ceph-mon[82388]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T03:45:59.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:58 smithi195 ceph-mon[82388]: Health check update: Degraded data redundancy: 21/399 objects degraded (5.263%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:46:00.017 INFO:tasks.rados.rados.0.smithi110.stdout:2401: finishing write tid 1 to smithi11075562-43 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:2401: finishing write tid 2 to smithi11075562-43 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:2401: finishing write tid 3 to smithi11075562-43 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1497 (ObjNum 751 snap 244 seq_num 751) dirty exists 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:2401: left oid 43 (ObjNum 751 snap 244 seq_num 751) 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:2401: done (1 left) 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:2402: done (0 left) 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:2403: copy_from oid 30 from oid 2 current snap is 244 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:2404: copy_from oid 24 from oid 16 current snap is 244 2024-09-14T03:46:00.018 INFO:tasks.rados.rados.0.smithi110.stdout:2405: delete oid 10 current snap is 244 2024-09-14T03:46:00.021 INFO:tasks.rados.rados.0.smithi110.stdout:2404: finishing copy_from to smithi11075562-24 2024-09-14T03:46:00.021 INFO:tasks.rados.rados.0.smithi110.stdout:2404: got expected ENOENT (src dne) 2024-09-14T03:46:00.021 INFO:tasks.rados.rados.0.smithi110.stdout:2404: finishing copy_from racing read to smithi11075562-24 2024-09-14T03:46:00.022 INFO:tasks.rados.rados.0.smithi110.stdout:2403: finishing copy_from to smithi11075562-30 2024-09-14T03:46:00.022 INFO:tasks.rados.rados.0.smithi110.stdout:2403: got expected ENOENT (src dne) 2024-09-14T03:46:00.022 INFO:tasks.rados.rados.0.smithi110.stdout:2403: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:46:00.025 INFO:tasks.rados.rados.0.smithi110.stdout:2403: done (2 left) 2024-09-14T03:46:00.025 INFO:tasks.rados.rados.0.smithi110.stdout:2404: done (1 left) 2024-09-14T03:46:00.025 INFO:tasks.rados.rados.0.smithi110.stdout:2405: done (0 left) 2024-09-14T03:46:00.025 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 244 2024-09-14T03:46:00.026 INFO:tasks.rados.rados.0.smithi110.stdout:2406: seq_num 753 ranges {794624=614400} 2024-09-14T03:46:00.034 INFO:tasks.rados.rados.0.smithi110.stdout:2406: writing smithi11075562-4 from 794624 to 1409024 tid 1 2024-09-14T03:46:00.034 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 244 2024-09-14T03:46:00.034 INFO:tasks.rados.rados.0.smithi110.stdout:2407: seq_num 754 ranges {0=630784} 2024-09-14T03:46:00.039 INFO:tasks.rados.rados.0.smithi110.stdout:2407: writing smithi11075562-10 from 0 to 630784 tid 1 2024-09-14T03:46:00.039 INFO:tasks.rados.rados.0.smithi110.stdout:2408: read oid 46 snap -1 2024-09-14T03:46:00.039 INFO:tasks.rados.rados.0.smithi110.stdout:2408: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:46:00.039 INFO:tasks.rados.rados.0.smithi110.stdout:2409: copy_from oid 23 from oid 16 current snap is 244 2024-09-14T03:46:00.039 INFO:tasks.rados.rados.0.smithi110.stdout:2410: read oid 2 snap -1 2024-09-14T03:46:00.039 INFO:tasks.rados.rados.0.smithi110.stdout:2410: expect deleted 2024-09-14T03:46:00.039 INFO:tasks.rados.rados.0.smithi110.stdout:2411: setattr oid 24 current snap is 244 2024-09-14T03:46:00.040 INFO:tasks.rados.rados.0.smithi110.stdout:2410: done (5 left) 2024-09-14T03:46:00.040 INFO:tasks.rados.rados.0.smithi110.stdout:2412: setattr oid 28 current snap is 244 2024-09-14T03:46:00.040 INFO:tasks.rados.rados.0.smithi110.stdout:2406: finishing write tid 1 to smithi11075562-4 2024-09-14T03:46:00.041 INFO:tasks.rados.rados.0.smithi110.stdout:2413: rmattr oid 36 current snap is 244 2024-09-14T03:46:00.041 INFO:tasks.rados.rados.0.smithi110.stdout:2413: done (6 left) 2024-09-14T03:46:00.041 INFO:tasks.rados.rados.0.smithi110.stdout:2414: read oid 45 snap -1 2024-09-14T03:46:00.041 INFO:tasks.rados.rados.0.smithi110.stdout:2414: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:46:00.041 INFO:tasks.rados.rados.0.smithi110.stdout:2415: read oid 7 snap -1 2024-09-14T03:46:00.042 INFO:tasks.rados.rados.0.smithi110.stdout:2415: expect (ObjNum 738 snap 235 seq_num 738) 2024-09-14T03:46:00.042 INFO:tasks.rados.rados.0.smithi110.stdout:2416: copy_from oid 22 from oid 9 current snap is 244 2024-09-14T03:46:00.042 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 34 current snap is 244 2024-09-14T03:46:00.042 INFO:tasks.rados.rados.0.smithi110.stdout:2417: seq_num 755 ranges {0=647168} 2024-09-14T03:46:00.045 INFO:tasks.rados.rados.0.smithi110.stdout:2417: writing smithi11075562-34 from 0 to 647168 tid 1 2024-09-14T03:46:00.045 INFO:tasks.rados.rados.0.smithi110.stdout:2418: snap_remove snap 241 2024-09-14T03:46:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: osdmap e558: 8 total, 8 up, 8 in 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: Standby daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: fsmap cephfs:1/1 {0=b=up:replay} 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[88262]: pgmap v466: 121 pgs: 121 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 256 KiB/s wr, 20 op/s; 9.6 MiB/s, 2 keys/s, 18 objects/s recovering 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:46:00.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:46:00.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: osdmap e558: 8 total, 8 up, 8 in 2024-09-14T03:46:00.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: Standby daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-14T03:46:00.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:46:00.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:46:00.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: fsmap cephfs:1/1 {0=b=up:replay} 2024-09-14T03:46:00.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:45:59 smithi110 ceph-mon[86423]: pgmap v466: 121 pgs: 121 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 256 KiB/s wr, 20 op/s; 9.6 MiB/s, 2 keys/s, 18 objects/s recovering 2024-09-14T03:46:00.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T03:46:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:46:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:46:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: osdmap e558: 8 total, 8 up, 8 in 2024-09-14T03:46:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: Standby daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-14T03:46:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:46:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:46:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: fsmap cephfs:1/1 {0=b=up:replay} 2024-09-14T03:46:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:45:59 smithi195 ceph-mon[82388]: pgmap v466: 121 pgs: 121 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 256 KiB/s wr, 20 op/s; 9.6 MiB/s, 2 keys/s, 18 objects/s recovering 2024-09-14T03:46:00.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:46:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:46:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:46:01.020 INFO:tasks.rados.rados.0.smithi110.stdout:2406: finishing write tid 2 to smithi11075562-4 2024-09-14T03:46:01.020 INFO:tasks.rados.rados.0.smithi110.stdout:2406: finishing write tid 3 to smithi11075562-4 2024-09-14T03:46:01.020 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1225 (ObjNum 753 snap 244 seq_num 753) dirty exists 2024-09-14T03:46:01.020 INFO:tasks.rados.rados.0.smithi110.stdout:2406: left oid 4 (ObjNum 753 snap 244 seq_num 753) 2024-09-14T03:46:01.020 INFO:tasks.rados.rados.0.smithi110.stdout:2409: finishing copy_from to smithi11075562-23 2024-09-14T03:46:01.020 INFO:tasks.rados.rados.0.smithi110.stdout:2409: got expected ENOENT (src dne) 2024-09-14T03:46:01.020 INFO:tasks.rados.rados.0.smithi110.stdout:2409: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:46:01.021 INFO:tasks.rados.rados.0.smithi110.stdout:2415: expect (ObjNum 738 snap 235 seq_num 738) 2024-09-14T03:46:01.037 INFO:tasks.rados.rados.0.smithi110.stdout:2407: finishing write tid 1 to smithi11075562-10 2024-09-14T03:46:01.037 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 804 (ObjNum 600 snap 183 seq_num 600) dirty exists 2024-09-14T03:46:01.037 INFO:tasks.rados.rados.0.smithi110.stdout:2406: done (10 left) 2024-09-14T03:46:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:2409: done (9 left) 2024-09-14T03:46:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:2412: done (8 left) 2024-09-14T03:46:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:2415: done (7 left) 2024-09-14T03:46:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:2418: done (6 left) 2024-09-14T03:46:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 244 2024-09-14T03:46:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:2407: finishing write tid 2 to smithi11075562-10 2024-09-14T03:46:01.038 INFO:tasks.rados.rados.0.smithi110.stdout:2419: seq_num 756 ranges {3702784=671744} 2024-09-14T03:46:01.061 INFO:tasks.rados.rados.0.smithi110.stdout:2419: writing smithi11075562-14 from 3702784 to 4374528 tid 1 2024-09-14T03:46:01.061 INFO:tasks.rados.rados.0.smithi110.stdout:2420: rollback oid 11 current snap is 244 2024-09-14T03:46:01.061 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 233 2024-09-14T03:46:01.061 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 244 2024-09-14T03:46:01.061 INFO:tasks.rados.rados.0.smithi110.stdout:2421: seq_num 757 ranges {2334720=688128} 2024-09-14T03:46:01.077 INFO:tasks.rados.rados.0.smithi110.stdout:2421: writing smithi11075562-44 from 2334720 to 3022848 tid 1 2024-09-14T03:46:01.077 INFO:tasks.rados.rados.0.smithi110.stdout:2422: snap_create 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1492 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2416: finishing copy_from to smithi11075562-22 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2416: got expected ENOENT (src dne) 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2416: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2407: finishing write tid 3 to smithi11075562-10 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1491 (ObjNum 754 snap 244 seq_num 754) dirty exists 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2407: left oid 10 (ObjNum 754 snap 244 seq_num 754) 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2417: finishing write tid 1 to smithi11075562-34 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2417: finishing write tid 2 to smithi11075562-34 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2417: finishing write tid 3 to smithi11075562-34 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1227 (ObjNum 755 snap 244 seq_num 755) dirty exists 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2417: left oid 34 (ObjNum 755 snap 244 seq_num 755) 2024-09-14T03:46:01.078 INFO:tasks.rados.rados.0.smithi110.stdout:2408: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:46:01.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:00 smithi110 ceph-mon[88262]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/399 objects degraded (5.263%), 6 pgs degraded) 2024-09-14T03:46:01.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:00 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:reconnect 2024-09-14T03:46:01.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:00 smithi110 ceph-mon[88262]: fsmap cephfs:1/1 {0=b=up:reconnect} 2024-09-14T03:46:01.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:00 smithi110 ceph-mon[88262]: osdmap e559: 8 total, 8 up, 8 in 2024-09-14T03:46:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:00 smithi110 ceph-mon[86423]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/399 objects degraded (5.263%), 6 pgs degraded) 2024-09-14T03:46:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:00 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:reconnect 2024-09-14T03:46:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:00 smithi110 ceph-mon[86423]: fsmap cephfs:1/1 {0=b=up:reconnect} 2024-09-14T03:46:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:00 smithi110 ceph-mon[86423]: osdmap e559: 8 total, 8 up, 8 in 2024-09-14T03:46:01.150 INFO:tasks.rados.rados.0.smithi110.stdout:2414: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:46:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:00 smithi195 ceph-mon[82388]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/399 objects degraded (5.263%), 6 pgs degraded) 2024-09-14T03:46:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:00 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:reconnect 2024-09-14T03:46:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:00 smithi195 ceph-mon[82388]: fsmap cephfs:1/1 {0=b=up:reconnect} 2024-09-14T03:46:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:00 smithi195 ceph-mon[82388]: osdmap e559: 8 total, 8 up, 8 in 2024-09-14T03:46:01.220 INFO:tasks.rados.rados.0.smithi110.stdout:2420: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 0 (ObjNum 65 snap 0 seq_num 1499749742) dirty dne 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:2419: finishing write tid 1 to smithi11075562-14 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:2419: finishing write tid 2 to smithi11075562-14 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:2419: finishing write tid 3 to smithi11075562-14 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1499 (ObjNum 756 snap 244 seq_num 756) dirty exists 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:2419: left oid 14 (ObjNum 756 snap 244 seq_num 756) 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:2421: finishing write tid 1 to smithi11075562-44 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:2421: finishing write tid 2 to smithi11075562-44 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:2421: finishing write tid 3 to smithi11075562-44 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 412 (ObjNum 757 snap 244 seq_num 757) dirty exists 2024-09-14T03:46:01.221 INFO:tasks.rados.rados.0.smithi110.stdout:2421: left oid 44 (ObjNum 757 snap 244 seq_num 757) 2024-09-14T03:46:01.533 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:46:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:46:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:46:01] "GET /metrics HTTP/1.1" 200 36181 "" "Prometheus/2.51.0" 2024-09-14T03:46:02.023 INFO:tasks.rados.rados.0.smithi110.stdout:2407: done (9 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2408: done (8 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2411: done (7 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2414: done (6 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2416: done (5 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2417: done (4 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2419: done (3 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2420: done (2 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2421: done (1 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2422: done (0 left) 2024-09-14T03:46:02.024 INFO:tasks.rados.rados.0.smithi110.stdout:2423: snap_remove snap 236 2024-09-14T03:46:02.089 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:46:02.089 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (6m) 2m ago 10m 19.5M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:46:02.089 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (6m) 10s ago 10m 84.6M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (12m) 2m ago 12m 20.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 ddaaf776057f 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (12m) 10s ago 12m 17.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 d306b18ee20b 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (5m) 10s ago 15m 455M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (6m) 2m ago 17m 525M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (5m) 2m ago 17m 55.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (4m) 10s ago 15m 49.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (5m) 2m ago 15m 49.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (6m) 2m ago 10m 9.92M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (6m) 10s ago 10m 11.0M - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (4m) 2m ago 15m 218M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (3m) 2m ago 14m 117M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (3m) 2m ago 14m 101M 2480M 19.3.0-4907-ga2782315 8801d2529e43 0cdc58c38711 2024-09-14T03:46:02.090 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (2m) 2m ago 14m 14.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2283ebcb5f10 2024-09-14T03:46:02.091 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (117s) 10s ago 13m 201M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6921b0a30047 2024-09-14T03:46:02.091 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (83s) 10s ago 13m 158M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8fe7f08d5882 2024-09-14T03:46:02.091 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (49s) 10s ago 12m 96.6M 2480M 19.3.0-4907-ga2782315 8801d2529e43 5305ca81bc26 2024-09-14T03:46:02.091 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (15s) 10s ago 12m 12.2M 2480M 19.3.0-4907-ga2782315 8801d2529e43 98e6e5252ce4 2024-09-14T03:46:02.091 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (5m) 10s ago 11m 54.8M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:46:02.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:rejoin 2024-09-14T03:46:02.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[86423]: fsmap cephfs:1/1 {0=b=up:rejoin} 2024-09-14T03:46:02.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[86423]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-14T03:46:02.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[86423]: osdmap e560: 8 total, 8 up, 8 in 2024-09-14T03:46:02.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[86423]: from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:02.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[86423]: pgmap v469: 121 pgs: 121 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.8 MiB/s wr, 30 op/s; 5.9 MiB/s, 2 keys/s, 17 objects/s recovering 2024-09-14T03:46:02.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:rejoin 2024-09-14T03:46:02.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[88262]: fsmap cephfs:1/1 {0=b=up:rejoin} 2024-09-14T03:46:02.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[88262]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-14T03:46:02.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[88262]: osdmap e560: 8 total, 8 up, 8 in 2024-09-14T03:46:02.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[88262]: from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:02.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:02 smithi110 ceph-mon[88262]: pgmap v469: 121 pgs: 121 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.8 MiB/s wr, 30 op/s; 5.9 MiB/s, 2 keys/s, 17 objects/s recovering 2024-09-14T03:46:02.438 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:46:02.438 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:46:02.438 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:46:02.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:46:02.438 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:46:02.439 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:46:02.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:46:02.439 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:46:02.439 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:46:02.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:46:02.439 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:46:02.439 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1 2024-09-14T03:46:02.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:46:02.440 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:46:02.440 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:46:02.440 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 13 2024-09-14T03:46:02.440 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:46:02.440 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:46:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:02 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:rejoin 2024-09-14T03:46:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:02 smithi195 ceph-mon[82388]: fsmap cephfs:1/1 {0=b=up:rejoin} 2024-09-14T03:46:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:02 smithi195 ceph-mon[82388]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-14T03:46:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:02 smithi195 ceph-mon[82388]: osdmap e560: 8 total, 8 up, 8 in 2024-09-14T03:46:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:02 smithi195 ceph-mon[82388]: from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:02 smithi195 ceph-mon[82388]: pgmap v469: 121 pgs: 121 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.8 MiB/s wr, 30 op/s; 5.9 MiB/s, 2 keys/s, 17 objects/s recovering 2024-09-14T03:46:02.739 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "mon", 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "osd" 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "13/20 daemons upgraded", 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading mds daemons", 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:46:02.740 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:46:03.026 INFO:tasks.rados.rados.0.smithi110.stdout:2423: done (0 left) 2024-09-14T03:46:03.026 INFO:tasks.rados.rados.0.smithi110.stdout:2424: read oid 16 snap -1 2024-09-14T03:46:03.027 INFO:tasks.rados.rados.0.smithi110.stdout:2424: expect deleted 2024-09-14T03:46:03.027 INFO:tasks.rados.rados.0.smithi110.stdout:2425: read oid 15 snap -1 2024-09-14T03:46:03.027 INFO:tasks.rados.rados.0.smithi110.stdout:2425: expect deleted 2024-09-14T03:46:03.027 INFO:tasks.rados.rados.0.smithi110.stdout:2426: snap_create 2024-09-14T03:46:03.344 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.344 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:active 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: fsmap cephfs:1 {0=b=up:active} 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: osdmap e561: 8 total, 8 up, 8 in 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/935563286' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[86423]: from='client.44429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:46:03.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:active 2024-09-14T03:46:03.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: fsmap cephfs:1 {0=b=up:active} 2024-09-14T03:46:03.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: osdmap e561: 8 total, 8 up, 8 in 2024-09-14T03:46:03.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/935563286' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:03.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:03 smithi110 ceph-mon[88262]: from='client.44429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.195:6832/1180864653,v1:172.21.15.195:6833/1180864653] up:active 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: fsmap cephfs:1 {0=b=up:active} 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: osdmap e561: 8 total, 8 up, 8 in 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/935563286' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:03 smithi195 ceph-mon[82388]: from='client.44429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:03.746 INFO:tasks.rados.rados.0.smithi110.stdout:2424: done (2 left) 2024-09-14T03:46:03.746 INFO:tasks.rados.rados.0.smithi110.stdout:2425: done (1 left) 2024-09-14T03:46:03.746 INFO:tasks.rados.rados.0.smithi110.stdout:2426: done (0 left) 2024-09-14T03:46:03.746 INFO:tasks.rados.rados.0.smithi110.stdout:2427: setattr oid 33 current snap is 246 2024-09-14T03:46:03.747 INFO:tasks.rados.rados.0.smithi110.stdout:2428: delete oid 35 current snap is 246 2024-09-14T03:46:03.752 INFO:tasks.rados.rados.0.smithi110.stdout:2428: done (1 left) 2024-09-14T03:46:03.752 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 246 2024-09-14T03:46:03.753 INFO:tasks.rados.rados.0.smithi110.stdout:2429: seq_num 759 ranges {2457600=720896} 2024-09-14T03:46:03.767 INFO:tasks.rados.rados.0.smithi110.stdout:2429: writing smithi11075562-49 from 2457600 to 3178496 tid 1 2024-09-14T03:46:03.767 INFO:tasks.rados.rados.0.smithi110.stdout:2430: snap_create 2024-09-14T03:46:03.768 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1568 (ObjNum 701 snap 224 seq_num 701) dirty exists 2024-09-14T03:46:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2429: finishing write tid 1 to smithi11075562-49 2024-09-14T03:46:03.772 INFO:tasks.rados.rados.0.smithi110.stdout:2429: finishing write tid 2 to smithi11075562-49 2024-09-14T03:46:03.773 INFO:tasks.rados.rados.0.smithi110.stdout:2429: finishing write tid 3 to smithi11075562-49 2024-09-14T03:46:03.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1467 (ObjNum 759 snap 246 seq_num 759) dirty exists 2024-09-14T03:46:03.773 INFO:tasks.rados.rados.0.smithi110.stdout:2429: left oid 49 (ObjNum 759 snap 246 seq_num 759) 2024-09-14T03:46:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:04 smithi110 ceph-mon[86423]: osdmap e562: 8 total, 8 up, 8 in 2024-09-14T03:46:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:04 smithi110 ceph-mon[86423]: osdmap e563: 8 total, 8 up, 8 in 2024-09-14T03:46:04.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:04 smithi110 ceph-mon[86423]: pgmap v473: 121 pgs: 121 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-09-14T03:46:04.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:04 smithi110 ceph-mon[88262]: osdmap e562: 8 total, 8 up, 8 in 2024-09-14T03:46:04.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:04 smithi110 ceph-mon[88262]: osdmap e563: 8 total, 8 up, 8 in 2024-09-14T03:46:04.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:04 smithi110 ceph-mon[88262]: pgmap v473: 121 pgs: 121 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-09-14T03:46:04.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:04 smithi195 ceph-mon[82388]: osdmap e562: 8 total, 8 up, 8 in 2024-09-14T03:46:04.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:04 smithi195 ceph-mon[82388]: osdmap e563: 8 total, 8 up, 8 in 2024-09-14T03:46:04.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:04 smithi195 ceph-mon[82388]: pgmap v473: 121 pgs: 121 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-09-14T03:46:04.751 INFO:tasks.rados.rados.0.smithi110.stdout:2427: done (2 left) 2024-09-14T03:46:04.751 INFO:tasks.rados.rados.0.smithi110.stdout:2429: done (1 left) 2024-09-14T03:46:04.751 INFO:tasks.rados.rados.0.smithi110.stdout:2430: done (0 left) 2024-09-14T03:46:04.752 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 247 2024-09-14T03:46:04.752 INFO:tasks.rados.rados.0.smithi110.stdout:2431: seq_num 760 ranges {2940928=737280} 2024-09-14T03:46:04.771 INFO:tasks.rados.rados.0.smithi110.stdout:2431: writing smithi11075562-32 from 2940928 to 3678208 tid 1 2024-09-14T03:46:04.771 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 247 2024-09-14T03:46:04.771 INFO:tasks.rados.rados.0.smithi110.stdout:2432: seq_num 761 ranges {0=753664} 2024-09-14T03:46:04.776 INFO:tasks.rados.rados.0.smithi110.stdout:2432: writing smithi11075562-12 from 0 to 753664 tid 1 2024-09-14T03:46:04.776 INFO:tasks.rados.rados.0.smithi110.stdout:2433: rmattr oid 2 current snap is 247 2024-09-14T03:46:04.776 INFO:tasks.rados.rados.0.smithi110.stdout:2433: done (2 left) 2024-09-14T03:46:04.776 INFO:tasks.rados.rados.0.smithi110.stdout:2434: rollback oid 46 current snap is 247 2024-09-14T03:46:04.776 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 46 to 246 2024-09-14T03:46:04.776 INFO:tasks.rados.rados.0.smithi110.stdout:2435: delete oid 16 current snap is 247 2024-09-14T03:46:04.777 INFO:tasks.rados.rados.0.smithi110.stdout:2431: finishing write tid 1 to smithi11075562-32 2024-09-14T03:46:04.777 INFO:tasks.rados.rados.0.smithi110.stdout:2431: finishing write tid 2 to smithi11075562-32 2024-09-14T03:46:04.778 INFO:tasks.rados.rados.0.smithi110.stdout:2431: finishing write tid 3 to smithi11075562-32 2024-09-14T03:46:04.778 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1449 (ObjNum 760 snap 247 seq_num 760) dirty exists 2024-09-14T03:46:04.778 INFO:tasks.rados.rados.0.smithi110.stdout:2431: left oid 32 (ObjNum 760 snap 247 seq_num 760) 2024-09-14T03:46:04.784 INFO:tasks.rados.rados.0.smithi110.stdout:2434: finishing rollback tid 0 to smithi11075562-46 2024-09-14T03:46:04.784 INFO:tasks.rados.rados.0.smithi110.stdout:2434: finishing rollback tid 1 to smithi11075562-46 2024-09-14T03:46:04.784 INFO:tasks.rados.rados.0.smithi110.stdout:2434: finishing rollback tid 2 to smithi11075562-46 2024-09-14T03:46:04.784 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1571 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:46:04.784 INFO:tasks.rados.rados.0.smithi110.stdout:2431: done (3 left) 2024-09-14T03:46:04.785 INFO:tasks.rados.rados.0.smithi110.stdout:2434: done (2 left) 2024-09-14T03:46:04.785 INFO:tasks.rados.rados.0.smithi110.stdout:2435: done (1 left) 2024-09-14T03:46:04.785 INFO:tasks.rados.rados.0.smithi110.stdout:2436: setattr oid 47 current snap is 247 2024-09-14T03:46:04.785 INFO:tasks.rados.rados.0.smithi110.stdout:2432: finishing write tid 1 to smithi11075562-12 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:2432: finishing write tid 2 to smithi11075562-12 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:2437: rmattr oid 48 current snap is 247 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:2437: done (2 left) 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:2438: read oid 14 snap -1 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:2438: expect (ObjNum 756 snap 244 seq_num 756) 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:2439: rollback oid 48 current snap is 247 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 48 to 235 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:2440: rollback oid 27 current snap is 247 2024-09-14T03:46:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 246 2024-09-14T03:46:04.787 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 247 2024-09-14T03:46:04.787 INFO:tasks.rados.rados.0.smithi110.stdout:2441: seq_num 763 ranges {5128192=786432} 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:2441: writing smithi11075562-30 from 5128192 to 5914624 tid 1 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:2442: copy_from oid 23 from oid 18 current snap is 247 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:2439: finishing rollback tid 1 to smithi11075562-48 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 0 (ObjNum 33 snap 0 seq_num 64993792) dirty dne 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:2440: finishing rollback tid 0 to smithi11075562-27 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:2440: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:2432: finishing write tid 3 to smithi11075562-12 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1706 (ObjNum 761 snap 247 seq_num 761) dirty exists 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:2432: left oid 12 (ObjNum 761 snap 247 seq_num 761) 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1708 (ObjNum 674 snap 209 seq_num 674) dirty exists 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:2440: finishing rollback tid 2 to smithi11075562-27 2024-09-14T03:46:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1570 (ObjNum 749 snap 243 seq_num 749) dirty exists 2024-09-14T03:46:04.820 INFO:tasks.rados.rados.0.smithi110.stdout:2438: expect (ObjNum 756 snap 244 seq_num 756) 2024-09-14T03:46:05.044 INFO:tasks.rados.rados.0.smithi110.stdout:2441: finishing write tid 1 to smithi11075562-30 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2441: finishing write tid 2 to smithi11075562-30 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2442: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2441: finishing write tid 3 to smithi11075562-30 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1452 (ObjNum 763 snap 247 seq_num 763) dirty exists 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2441: left oid 30 (ObjNum 763 snap 247 seq_num 763) 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2442: finishing copy_from to smithi11075562-23 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1448 (ObjNum 65 snap 0 seq_num 1532916318) dirty exists 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2432: done (6 left) 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2436: done (5 left) 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2438: done (4 left) 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2439: done (3 left) 2024-09-14T03:46:05.045 INFO:tasks.rados.rados.0.smithi110.stdout:2440: done (2 left) 2024-09-14T03:46:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:2441: done (1 left) 2024-09-14T03:46:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:2442: done (0 left) 2024-09-14T03:46:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:2443: rollback oid 20 current snap is 247 2024-09-14T03:46:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 239 2024-09-14T03:46:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:2444: delete oid 22 current snap is 247 2024-09-14T03:46:05.046 INFO:tasks.rados.rados.0.smithi110.stdout:2444: done (1 left) 2024-09-14T03:46:05.047 INFO:tasks.rados.rados.0.smithi110.stdout:2445: read oid 21 snap 242 2024-09-14T03:46:05.047 INFO:tasks.rados.rados.0.smithi110.stdout:2445: expect deleted 2024-09-14T03:46:05.047 INFO:tasks.rados.rados.0.smithi110.stdout:2446: snap_create 2024-09-14T03:46:05.048 INFO:tasks.rados.rados.0.smithi110.stdout:2443: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:46:05.048 INFO:tasks.rados.rados.0.smithi110.stdout:2443: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:46:05.048 INFO:tasks.rados.rados.0.smithi110.stdout:2443: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:46:05.048 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1574 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:46:05.400 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:05 smithi110 podman[111392]: 2024-09-14 03:46:05.103792756 +0000 UTC m=+6.207500004 container died ddaaf776057f40c5f10bdb1b61179f20b5943d68c73b28194aaa0d01ec75e0bb (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-14T03:46:05.684 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:05 smithi110 podman[111392]: 2024-09-14 03:46:05.515764421 +0000 UTC m=+6.619471657 container cleanup ddaaf776057f40c5f10bdb1b61179f20b5943d68c73b28194aaa0d01ec75e0bb (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-09-14T03:46:05.684 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:05 smithi110 bash[111392]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a 2024-09-14T03:46:05.756 INFO:tasks.rados.rados.0.smithi110.stdout:2443: done (2 left) 2024-09-14T03:46:05.756 INFO:tasks.rados.rados.0.smithi110.stdout:2445: done (1 left) 2024-09-14T03:46:05.757 INFO:tasks.rados.rados.0.smithi110.stdout:2446: done (0 left) 2024-09-14T03:46:05.757 INFO:tasks.rados.rados.0.smithi110.stdout:2447: read oid 21 snap -1 2024-09-14T03:46:05.757 INFO:tasks.rados.rados.0.smithi110.stdout:2447: expect deleted 2024-09-14T03:46:05.757 INFO:tasks.rados.rados.0.smithi110.stdout:2448: read oid 9 snap 245 2024-09-14T03:46:05.757 INFO:tasks.rados.rados.0.smithi110.stdout:2448: expect deleted 2024-09-14T03:46:05.757 INFO:tasks.rados.rados.0.smithi110.stdout:2449: read oid 28 snap -1 2024-09-14T03:46:05.757 INFO:tasks.rados.rados.0.smithi110.stdout:2449: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:46:05.757 INFO:tasks.rados.rados.0.smithi110.stdout:2450: snap_create 2024-09-14T03:46:05.759 INFO:tasks.rados.rados.0.smithi110.stdout:2449: expect (ObjNum 600 snap 183 seq_num 600) 2024-09-14T03:46:05.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:05 smithi110 ceph-mon[88262]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T03:46:05.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:05 smithi110 ceph-mon[86423]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T03:46:05.946 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:05 smithi110 podman[111533]: 2024-09-14 03:46:05.68374534 +0000 UTC m=+0.577438288 container remove ddaaf776057f40c5f10bdb1b61179f20b5943d68c73b28194aaa0d01ec75e0bb (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-14T03:46:05.946 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:05 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.a.service: Deactivated successfully. 2024-09-14T03:46:05.946 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:05 smithi110 systemd[1]: Stopped Ceph mds.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:46:05.947 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:05 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.a.service: Consumed 1.608s CPU time. 2024-09-14T03:46:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:05 smithi195 ceph-mon[82388]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T03:46:06.433 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:06 smithi110 systemd[1]: Starting Ceph mds.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:46:06.433 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:06 smithi110 podman[111661]: 2024-09-14 03:46:06.21946955 +0000 UTC m=+0.015664336 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:46:06.433 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:06 smithi110 podman[111661]: 2024-09-14 03:46:06.32646066 +0000 UTC m=+0.122655505 container create ddd26c7ac6c28264ba316d5148cfda05d9bc868edc0cafc3ce03b0b015368bd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:46:06.696 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:06 smithi110 podman[111661]: 2024-09-14 03:46:06.54553571 +0000 UTC m=+0.341730497 container init ddd26c7ac6c28264ba316d5148cfda05d9bc868edc0cafc3ce03b0b015368bd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:46:06.696 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:06 smithi110 podman[111661]: 2024-09-14 03:46:06.54835757 +0000 UTC m=+0.344552365 container start ddd26c7ac6c28264ba316d5148cfda05d9bc868edc0cafc3ce03b0b015368bd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-14T03:46:06.696 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[111676]: starting mds.a at 2024-09-14T03:46:06.696 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:06 smithi110 bash[111661]: ddd26c7ac6c28264ba316d5148cfda05d9bc868edc0cafc3ce03b0b015368bd3 2024-09-14T03:46:06.696 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:06 smithi110 systemd[1]: Started Ceph mds.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:46:06.762 INFO:tasks.rados.rados.0.smithi110.stdout:2447: done (3 left) 2024-09-14T03:46:06.763 INFO:tasks.rados.rados.0.smithi110.stdout:2448: done (2 left) 2024-09-14T03:46:06.763 INFO:tasks.rados.rados.0.smithi110.stdout:2449: done (1 left) 2024-09-14T03:46:06.763 INFO:tasks.rados.rados.0.smithi110.stdout:2450: done (0 left) 2024-09-14T03:46:06.763 INFO:tasks.rados.rados.0.smithi110.stdout:2451: read oid 13 snap 240 2024-09-14T03:46:06.763 INFO:tasks.rados.rados.0.smithi110.stdout:2451: expect (ObjNum 727 snap 229 seq_num 727) 2024-09-14T03:46:06.763 INFO:tasks.rados.rados.0.smithi110.stdout:2452: delete oid 9 current snap is 249 2024-09-14T03:46:06.767 INFO:tasks.rados.rados.0.smithi110.stdout:2452: done (1 left) 2024-09-14T03:46:06.767 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 249 2024-09-14T03:46:06.767 INFO:tasks.rados.rados.0.smithi110.stdout:2453: seq_num 766 ranges {5152768=434176,5586944=401408} 2024-09-14T03:46:06.798 INFO:tasks.rados.rados.0.smithi110.stdout:2453: writing smithi11075562-5 from 5152768 to 5586944 tid 1 2024-09-14T03:46:06.800 INFO:tasks.rados.rados.0.smithi110.stdout:2453: writing smithi11075562-5 from 5586944 to 5988352 tid 2 2024-09-14T03:46:06.800 INFO:tasks.rados.rados.0.smithi110.stdout:2454: delete oid 46 current snap is 249 2024-09-14T03:46:06.801 INFO:tasks.rados.rados.0.smithi110.stdout:2451: expect (ObjNum 727 snap 229 seq_num 727) 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:2453: finishing write tid 1 to smithi11075562-5 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:2453: finishing write tid 2 to smithi11075562-5 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:2451: done (2 left) 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:2454: done (1 left) 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:2455: snap_create 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:2453: finishing write tid 3 to smithi11075562-5 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:2453: finishing write tid 4 to smithi11075562-5 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1712 (ObjNum 766 snap 249 seq_num 766) dirty exists 2024-09-14T03:46:07.194 INFO:tasks.rados.rados.0.smithi110.stdout:2453: left oid 5 (ObjNum 766 snap 249 seq_num 766) 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[88262]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[88262]: pgmap v476: 121 pgs: 121 active+clean; 185 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.7 MiB/s wr, 42 op/s 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[86423]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[86423]: pgmap v476: 121 pgs: 121 active+clean; 185 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.7 MiB/s wr, 42 op/s 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:07.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:06 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:07.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:06 smithi195 ceph-mon[82388]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T03:46:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:06 smithi195 ceph-mon[82388]: pgmap v476: 121 pgs: 121 active+clean; 185 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.7 MiB/s wr, 42 op/s 2024-09-14T03:46:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:06 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:06 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:06 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:08.130 INFO:tasks.rados.rados.0.smithi110.stdout:2453: done (1 left) 2024-09-14T03:46:08.130 INFO:tasks.rados.rados.0.smithi110.stdout:2455: done (0 left) 2024-09-14T03:46:08.130 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 250 2024-09-14T03:46:08.130 INFO:tasks.rados.rados.0.smithi110.stdout:2456: seq_num 768 ranges {5652480=466944,6119424=401408} 2024-09-14T03:46:08.161 INFO:tasks.rados.rados.0.smithi110.stdout:2456: writing smithi11075562-13 from 5652480 to 6119424 tid 1 2024-09-14T03:46:08.164 INFO:tasks.rados.rados.0.smithi110.stdout:2456: writing smithi11075562-13 from 6119424 to 6520832 tid 2 2024-09-14T03:46:08.164 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 250 2024-09-14T03:46:08.164 INFO:tasks.rados.rados.0.smithi110.stdout:2457: seq_num 769 ranges {3637248=483328,4120576=401408} 2024-09-14T03:46:08.186 INFO:tasks.rados.rados.0.smithi110.stdout:2457: writing smithi11075562-17 from 3637248 to 4120576 tid 1 2024-09-14T03:46:08.188 INFO:tasks.rados.rados.0.smithi110.stdout:2457: writing smithi11075562-17 from 4120576 to 4521984 tid 2 2024-09-14T03:46:08.189 INFO:tasks.rados.rados.0.smithi110.stdout:2458: read oid 30 snap -1 2024-09-14T03:46:08.189 INFO:tasks.rados.rados.0.smithi110.stdout:2458: expect (ObjNum 763 snap 247 seq_num 763) 2024-09-14T03:46:08.189 INFO:tasks.rados.rados.0.smithi110.stdout:2459: snap_remove snap 240 2024-09-14T03:46:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[88262]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T03:46:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:boot 2024-09-14T03:46:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[88262]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-14T03:46:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:46:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[86423]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T03:46:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:boot 2024-09-14T03:46:08.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[86423]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-14T03:46:08.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:46:08.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:07 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:07 smithi195 ceph-mon[82388]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T03:46:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:07 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:boot 2024-09-14T03:46:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:07 smithi195 ceph-mon[82388]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-14T03:46:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:07 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:46:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:07 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:08.747 INFO:tasks.rados.rados.0.smithi110.stdout:2459: done (3 left) 2024-09-14T03:46:08.748 INFO:tasks.rados.rados.0.smithi110.stdout:2460: delete oid 47 current snap is 250 2024-09-14T03:46:08.748 INFO:tasks.rados.rados.0.smithi110.stdout:2456: finishing write tid 1 to smithi11075562-13 2024-09-14T03:46:08.748 INFO:tasks.rados.rados.0.smithi110.stdout:2456: finishing write tid 2 to smithi11075562-13 2024-09-14T03:46:08.748 INFO:tasks.rados.rados.0.smithi110.stdout:2456: finishing write tid 3 to smithi11075562-13 2024-09-14T03:46:08.748 INFO:tasks.rados.rados.0.smithi110.stdout:2456: finishing write tid 4 to smithi11075562-13 2024-09-14T03:46:08.748 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1470 (ObjNum 768 snap 250 seq_num 768) dirty exists 2024-09-14T03:46:08.748 INFO:tasks.rados.rados.0.smithi110.stdout:2456: left oid 13 (ObjNum 768 snap 250 seq_num 768) 2024-09-14T03:46:08.748 INFO:tasks.rados.rados.0.smithi110.stdout:2457: finishing write tid 1 to smithi11075562-17 2024-09-14T03:46:08.749 INFO:tasks.rados.rados.0.smithi110.stdout:2457: finishing write tid 2 to smithi11075562-17 2024-09-14T03:46:08.749 INFO:tasks.rados.rados.0.smithi110.stdout:2457: finishing write tid 3 to smithi11075562-17 2024-09-14T03:46:08.749 INFO:tasks.rados.rados.0.smithi110.stdout:2457: finishing write tid 4 to smithi11075562-17 2024-09-14T03:46:08.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1142 (ObjNum 769 snap 250 seq_num 769) dirty exists 2024-09-14T03:46:08.749 INFO:tasks.rados.rados.0.smithi110.stdout:2457: left oid 17 (ObjNum 769 snap 250 seq_num 769) 2024-09-14T03:46:08.749 INFO:tasks.rados.rados.0.smithi110.stdout:2458: expect (ObjNum 763 snap 247 seq_num 763) 2024-09-14T03:46:09.056 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:08 smithi110 ceph-mon[88262]: pgmap v478: 121 pgs: 121 active+clean; 185 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 812 KiB/s wr, 27 op/s 2024-09-14T03:46:09.057 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:08 smithi110 ceph-mon[88262]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T03:46:09.057 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:08 smithi110 ceph-mon[88262]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T03:46:09.057 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:08 smithi110 ceph-mon[86423]: pgmap v478: 121 pgs: 121 active+clean; 185 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 812 KiB/s wr, 27 op/s 2024-09-14T03:46:09.057 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:08 smithi110 ceph-mon[86423]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T03:46:09.057 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:08 smithi110 ceph-mon[86423]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T03:46:09.064 INFO:tasks.rados.rados.0.smithi110.stdout:2456: done (3 left) 2024-09-14T03:46:09.064 INFO:tasks.rados.rados.0.smithi110.stdout:2457: done (2 left) 2024-09-14T03:46:09.064 INFO:tasks.rados.rados.0.smithi110.stdout:2458: done (1 left) 2024-09-14T03:46:09.064 INFO:tasks.rados.rados.0.smithi110.stdout:2460: done (0 left) 2024-09-14T03:46:09.064 INFO:tasks.rados.rados.0.smithi110.stdout:2461: snap_create 2024-09-14T03:46:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:08 smithi195 ceph-mon[82388]: pgmap v478: 121 pgs: 121 active+clean; 185 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 812 KiB/s wr, 27 op/s 2024-09-14T03:46:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:08 smithi195 ceph-mon[82388]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T03:46:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:08 smithi195 ceph-mon[82388]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T03:46:09.771 INFO:tasks.rados.rados.0.smithi110.stdout:2461: done (0 left) 2024-09-14T03:46:09.771 INFO:tasks.rados.rados.0.smithi110.stdout:2462: delete oid 28 current snap is 251 2024-09-14T03:46:09.773 INFO:tasks.rados.rados.0.smithi110.stdout:2462: done (0 left) 2024-09-14T03:46:09.773 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 251 2024-09-14T03:46:09.773 INFO:tasks.rados.rados.0.smithi110.stdout:2463: seq_num 772 ranges {14180352=532480,14712832=401408} 2024-09-14T03:46:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:2463: writing smithi11075562-39 from 14180352 to 14712832 tid 1 2024-09-14T03:46:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:2463: writing smithi11075562-39 from 14712832 to 15114240 tid 2 2024-09-14T03:46:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 251 2024-09-14T03:46:09.855 INFO:tasks.rados.rados.0.smithi110.stdout:2464: seq_num 773 ranges {6520832=548864,7069696=401408} 2024-09-14T03:46:09.893 INFO:tasks.rados.rados.0.smithi110.stdout:2464: writing smithi11075562-13 from 6520832 to 7069696 tid 1 2024-09-14T03:46:09.895 INFO:tasks.rados.rados.0.smithi110.stdout:2464: writing smithi11075562-13 from 7069696 to 7471104 tid 2 2024-09-14T03:46:09.895 INFO:tasks.rados.rados.0.smithi110.stdout:2465: snap_remove snap 235 2024-09-14T03:46:10.628 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[86423]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[86423]: pgmap v482: 121 pgs: 121 active+clean; 186 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 764 KiB/s wr, 14 op/s 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[88262]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[88262]: pgmap v482: 121 pgs: 121 active+clean; 186 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 764 KiB/s wr, 14 op/s 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:10 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:10 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:10 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:10 smithi195 ceph-mon[82388]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T03:46:10.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:10 smithi195 ceph-mon[82388]: pgmap v482: 121 pgs: 121 active+clean; 186 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 764 KiB/s wr, 14 op/s 2024-09-14T03:46:10.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:10 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:10 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:10.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:46:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:46:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:46:10.775 INFO:tasks.rados.rados.0.smithi110.stdout:2465: done (2 left) 2024-09-14T03:46:10.775 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 251 2024-09-14T03:46:10.776 INFO:tasks.rados.rados.0.smithi110.stdout:2466: seq_num 774 ranges {0=565248,565248=401408} 2024-09-14T03:46:10.782 INFO:tasks.rados.rados.0.smithi110.stdout:2466: writing smithi11075562-28 from 0 to 565248 tid 1 2024-09-14T03:46:10.782 INFO:tasks.rados.rados.0.smithi110.stdout:2466: writing smithi11075562-28 from 565248 to 966656 tid 2 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2467: read oid 31 snap 229 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2467: expect (ObjNum 623 snap 190 seq_num 623) 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2468: rollback oid 30 current snap is 251 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 244 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2463: finishing write tid 1 to smithi11075562-39 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2463: finishing write tid 2 to smithi11075562-39 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2463: finishing write tid 3 to smithi11075562-39 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2463: finishing write tid 4 to smithi11075562-39 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1502 (ObjNum 772 snap 251 seq_num 772) dirty exists 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2463: left oid 39 (ObjNum 772 snap 251 seq_num 772) 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2464: finishing write tid 1 to smithi11075562-13 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2464: finishing write tid 2 to smithi11075562-13 2024-09-14T03:46:10.783 INFO:tasks.rados.rados.0.smithi110.stdout:2464: finishing write tid 3 to smithi11075562-13 2024-09-14T03:46:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:2464: finishing write tid 4 to smithi11075562-13 2024-09-14T03:46:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1473 (ObjNum 773 snap 251 seq_num 773) dirty exists 2024-09-14T03:46:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:2464: left oid 13 (ObjNum 773 snap 251 seq_num 773) 2024-09-14T03:46:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:2463: done (4 left) 2024-09-14T03:46:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:2464: done (3 left) 2024-09-14T03:46:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:2469: copy_from oid 17 from oid 10 current snap is 251 2024-09-14T03:46:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 251 2024-09-14T03:46:10.784 INFO:tasks.rados.rados.0.smithi110.stdout:2470: seq_num 775 ranges {8822784=589824,9412608=401408} 2024-09-14T03:46:10.835 INFO:tasks.rados.rados.0.smithi110.stdout:2470: writing smithi11075562-6 from 8822784 to 9412608 tid 1 2024-09-14T03:46:10.837 INFO:tasks.rados.rados.0.smithi110.stdout:2470: writing smithi11075562-6 from 9412608 to 9814016 tid 2 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2471: delete oid 38 current snap is 251 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2466: finishing write tid 1 to smithi11075562-28 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2466: finishing write tid 2 to smithi11075562-28 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2468: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2468: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2468: finishing rollback tid 2 to smithi11075562-30 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1464 (ObjNum 687 snap 215 seq_num 687) dirty exists 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2466: finishing write tid 3 to smithi11075562-28 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2466: finishing write tid 4 to smithi11075562-28 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 817 (ObjNum 774 snap 251 seq_num 774) dirty exists 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2466: left oid 28 (ObjNum 774 snap 251 seq_num 774) 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2469: finishing copy_from racing read to smithi11075562-17 2024-09-14T03:46:10.838 INFO:tasks.rados.rados.0.smithi110.stdout:2469: finishing copy_from to smithi11075562-17 2024-09-14T03:46:10.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1491 (ObjNum 754 snap 244 seq_num 754) dirty exists 2024-09-14T03:46:10.839 INFO:tasks.rados.rados.0.smithi110.stdout:2467: expect (ObjNum 623 snap 190 seq_num 623) 2024-09-14T03:46:11.401 INFO:tasks.rados.rados.0.smithi110.stdout:2470: finishing write tid 1 to smithi11075562-6 2024-09-14T03:46:11.401 INFO:tasks.rados.rados.0.smithi110.stdout:2470: finishing write tid 2 to smithi11075562-6 2024-09-14T03:46:11.401 INFO:tasks.rados.rados.0.smithi110.stdout:2470: finishing write tid 3 to smithi11075562-6 2024-09-14T03:46:11.401 INFO:tasks.rados.rados.0.smithi110.stdout:2470: finishing write tid 4 to smithi11075562-6 2024-09-14T03:46:11.401 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1453 (ObjNum 775 snap 251 seq_num 775) dirty exists 2024-09-14T03:46:11.401 INFO:tasks.rados.rados.0.smithi110.stdout:2470: left oid 6 (ObjNum 775 snap 251 seq_num 775) 2024-09-14T03:46:11.402 INFO:tasks.rados.rados.0.smithi110.stdout:2466: done (5 left) 2024-09-14T03:46:11.402 INFO:tasks.rados.rados.0.smithi110.stdout:2467: done (4 left) 2024-09-14T03:46:11.402 INFO:tasks.rados.rados.0.smithi110.stdout:2468: done (3 left) 2024-09-14T03:46:11.402 INFO:tasks.rados.rados.0.smithi110.stdout:2469: done (2 left) 2024-09-14T03:46:11.402 INFO:tasks.rados.rados.0.smithi110.stdout:2470: done (1 left) 2024-09-14T03:46:11.402 INFO:tasks.rados.rados.0.smithi110.stdout:2471: done (0 left) 2024-09-14T03:46:11.402 INFO:tasks.rados.rados.0.smithi110.stdout:2472: delete oid 20 current snap is 251 2024-09-14T03:46:11.403 INFO:tasks.rados.rados.0.smithi110.stdout:2472: done (0 left) 2024-09-14T03:46:11.403 INFO:tasks.rados.rados.0.smithi110.stdout:2473: copy_from oid 34 from oid 33 current snap is 251 2024-09-14T03:46:11.403 INFO:tasks.rados.rados.0.smithi110.stdout:2474: snap_remove snap 242 2024-09-14T03:46:11.658 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:46:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:46:11] "GET /metrics HTTP/1.1" 200 36181 "" "Prometheus/2.51.0" 2024-09-14T03:46:11.779 INFO:tasks.rados.rados.0.smithi110.stdout:2474: done (1 left) 2024-09-14T03:46:11.780 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 251 2024-09-14T03:46:11.780 INFO:tasks.rados.rados.0.smithi110.stdout:2475: seq_num 778 ranges {0=638976,638976=401408} 2024-09-14T03:46:11.784 INFO:tasks.rados.rados.0.smithi110.stdout:2475: writing smithi11075562-19 from 0 to 638976 tid 1 2024-09-14T03:46:11.788 INFO:tasks.rados.rados.0.smithi110.stdout:2475: writing smithi11075562-19 from 638976 to 1040384 tid 2 2024-09-14T03:46:11.788 INFO:tasks.rados.rados.0.smithi110.stdout:2476: rmattr oid 4 current snap is 251 2024-09-14T03:46:11.788 INFO:tasks.rados.rados.0.smithi110.stdout:2476: done (2 left) 2024-09-14T03:46:11.788 INFO:tasks.rados.rados.0.smithi110.stdout:2477: rmattr oid 24 current snap is 251 2024-09-14T03:46:11.788 INFO:tasks.rados.rados.0.smithi110.stdout:2473: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:46:11.788 INFO:tasks.rados.rados.0.smithi110.stdout:2473: finishing copy_from to smithi11075562-34 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1568 (ObjNum 701 snap 224 seq_num 701) dirty exists 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:2473: done (2 left) 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:2478: rollback oid 7 current snap is 251 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 7 to 229 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:2479: rmattr oid 45 current snap is 251 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:2479: done (3 left) 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:2480: read oid 43 snap 246 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:2480: expect (ObjNum 751 snap 244 seq_num 751) 2024-09-14T03:46:11.789 INFO:tasks.rados.rados.0.smithi110.stdout:2481: snap_create 2024-09-14T03:46:11.795 INFO:tasks.rados.rados.0.smithi110.stdout:2475: finishing write tid 1 to smithi11075562-19 2024-09-14T03:46:11.797 INFO:tasks.rados.rados.0.smithi110.stdout:2475: finishing write tid 2 to smithi11075562-19 2024-09-14T03:46:11.798 INFO:tasks.rados.rados.0.smithi110.stdout:2478: finishing rollback tid 0 to smithi11075562-7 2024-09-14T03:46:11.798 INFO:tasks.rados.rados.0.smithi110.stdout:2475: finishing write tid 3 to smithi11075562-19 2024-09-14T03:46:11.798 INFO:tasks.rados.rados.0.smithi110.stdout:2478: finishing rollback tid 1 to smithi11075562-7 2024-09-14T03:46:11.799 INFO:tasks.rados.rados.0.smithi110.stdout:2478: finishing rollback tid 2 to smithi11075562-7 2024-09-14T03:46:11.799 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1580 (ObjNum 691 snap 216 seq_num 691) dirty exists 2024-09-14T03:46:11.799 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1493 (ObjNum 65 snap 0 seq_num 1935760478) dirty exists 2024-09-14T03:46:11.800 INFO:tasks.rados.rados.0.smithi110.stdout:2475: finishing write tid 4 to smithi11075562-19 2024-09-14T03:46:11.800 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 1456 (ObjNum 778 snap 251 seq_num 778) dirty exists 2024-09-14T03:46:11.801 INFO:tasks.rados.rados.0.smithi110.stdout:2475: left oid 19 (ObjNum 778 snap 251 seq_num 778) 2024-09-14T03:46:11.817 INFO:tasks.rados.rados.0.smithi110.stdout:2480: expect (ObjNum 751 snap 244 seq_num 751) 2024-09-14T03:46:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:11 smithi110 ceph-mon[88262]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T03:46:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:11 smithi110 ceph-mon[86423]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T03:46:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:11 smithi195 ceph-mon[82388]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T03:46:12.785 INFO:tasks.rados.rados.0.smithi110.stdout:2475: done (4 left) 2024-09-14T03:46:12.785 INFO:tasks.rados.rados.0.smithi110.stdout:2477: done (3 left) 2024-09-14T03:46:12.785 INFO:tasks.rados.rados.0.smithi110.stdout:2478: done (2 left) 2024-09-14T03:46:12.785 INFO:tasks.rados.rados.0.smithi110.stdout:2480: done (1 left) 2024-09-14T03:46:12.785 INFO:tasks.rados.rados.0.smithi110.stdout:2481: done (0 left) 2024-09-14T03:46:12.786 INFO:tasks.rados.rados.0.smithi110.stdout:2482: setattr oid 23 current snap is 252 2024-09-14T03:46:12.786 INFO:tasks.rados.rados.0.smithi110.stdout:2483: snap_create 2024-09-14T03:46:12.793 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1494 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-14T03:46:13.108 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:12 smithi110 ceph-mon[88262]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T03:46:13.108 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:12 smithi110 ceph-mon[88262]: pgmap v485: 121 pgs: 121 active+clean; 190 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-14T03:46:13.108 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:12 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:13.108 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:13.109 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:12 smithi110 ceph-mon[86423]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T03:46:13.109 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:12 smithi110 ceph-mon[86423]: pgmap v485: 121 pgs: 121 active+clean; 190 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-14T03:46:13.109 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:12 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:13.109 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:12 smithi195 ceph-mon[82388]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T03:46:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:12 smithi195 ceph-mon[82388]: pgmap v485: 121 pgs: 121 active+clean; 190 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-14T03:46:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:12 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2482: done (1 left) 2024-09-14T03:46:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2483: done (0 left) 2024-09-14T03:46:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2484: read oid 30 snap 231 2024-09-14T03:46:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2484: expect (ObjNum 687 snap 215 seq_num 687) 2024-09-14T03:46:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2485: read oid 25 snap -1 2024-09-14T03:46:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2485: expect (ObjNum 642 snap 199 seq_num 642) 2024-09-14T03:46:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 253 2024-09-14T03:46:13.748 INFO:tasks.rados.rados.0.smithi110.stdout:2486: seq_num 779 ranges {5087232=655360,5742592=401408} 2024-09-14T03:46:13.780 INFO:tasks.rados.rados.0.smithi110.stdout:2486: writing smithi11075562-1 from 5087232 to 5742592 tid 1 2024-09-14T03:46:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:2486: writing smithi11075562-1 from 5742592 to 6144000 tid 2 2024-09-14T03:46:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:2487: snap_create 2024-09-14T03:46:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:2484: expect (ObjNum 687 snap 215 seq_num 687) 2024-09-14T03:46:13.987 INFO:tasks.rados.rados.0.smithi110.stdout:2485: expect (ObjNum 642 snap 199 seq_num 642) 2024-09-14T03:46:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T03:46:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:46:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[88262]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:14.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:46:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:46:14.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:13 smithi110 ceph-mon[86423]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:46:14.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:13 smithi195 ceph-mon[82388]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T03:46:14.762 INFO:tasks.rados.rados.0.smithi110.stdout:2486: finishing write tid 1 to smithi11075562-1 2024-09-14T03:46:14.762 INFO:tasks.rados.rados.0.smithi110.stdout:2486: finishing write tid 2 to smithi11075562-1 2024-09-14T03:46:14.763 INFO:tasks.rados.rados.0.smithi110.stdout:2486: finishing write tid 3 to smithi11075562-1 2024-09-14T03:46:14.763 INFO:tasks.rados.rados.0.smithi110.stdout:2486: finishing write tid 4 to smithi11075562-1 2024-09-14T03:46:14.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1476 (ObjNum 779 snap 253 seq_num 779) dirty exists 2024-09-14T03:46:14.763 INFO:tasks.rados.rados.0.smithi110.stdout:2486: left oid 1 (ObjNum 779 snap 253 seq_num 779) 2024-09-14T03:46:14.789 INFO:tasks.rados.rados.0.smithi110.stdout:2484: done (3 left) 2024-09-14T03:46:14.789 INFO:tasks.rados.rados.0.smithi110.stdout:2485: done (2 left) 2024-09-14T03:46:14.790 INFO:tasks.rados.rados.0.smithi110.stdout:2486: done (1 left) 2024-09-14T03:46:14.790 INFO:tasks.rados.rados.0.smithi110.stdout:2487: done (0 left) 2024-09-14T03:46:14.790 INFO:tasks.rados.rados.0.smithi110.stdout:2488: delete oid 6 current snap is 254 2024-09-14T03:46:14.792 INFO:tasks.rados.rados.0.smithi110.stdout:2488: done (0 left) 2024-09-14T03:46:14.793 INFO:tasks.rados.rados.0.smithi110.stdout:2489: snap_remove snap 233 2024-09-14T03:46:15.098 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:14 smithi195 ceph-mon[82388]: Detected new or changed devices on smithi110 2024-09-14T03:46:15.098 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:14 smithi195 ceph-mon[82388]: Upgrade: It appears safe to stop mds.b 2024-09-14T03:46:15.098 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:14 smithi195 ceph-mon[82388]: pgmap v488: 121 pgs: 121 active+clean; 193 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T03:46:15.098 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:14 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:15.099 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:14 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:46:15.099 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:14 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:46:15.099 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:14 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[88262]: Detected new or changed devices on smithi110 2024-09-14T03:46:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[88262]: Upgrade: It appears safe to stop mds.b 2024-09-14T03:46:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[88262]: pgmap v488: 121 pgs: 121 active+clean; 193 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T03:46:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:46:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:46:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[86423]: Detected new or changed devices on smithi110 2024-09-14T03:46:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[86423]: Upgrade: It appears safe to stop mds.b 2024-09-14T03:46:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[86423]: pgmap v488: 121 pgs: 121 active+clean; 193 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T03:46:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:46:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:46:15.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:14 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:15.797 INFO:tasks.rados.rados.0.smithi110.stdout:2489: done (0 left) 2024-09-14T03:46:15.797 INFO:tasks.rados.rados.0.smithi110.stdout:2490: read oid 49 snap -1 2024-09-14T03:46:15.797 INFO:tasks.rados.rados.0.smithi110.stdout:2490: expect (ObjNum 759 snap 246 seq_num 759) 2024-09-14T03:46:15.797 INFO:tasks.rados.rados.0.smithi110.stdout:2491: rmattr oid 36 current snap is 254 2024-09-14T03:46:15.797 INFO:tasks.rados.rados.0.smithi110.stdout:2491: done (1 left) 2024-09-14T03:46:15.797 INFO:tasks.rados.rados.0.smithi110.stdout:2492: rmattr oid 45 current snap is 254 2024-09-14T03:46:15.797 INFO:tasks.rados.rados.0.smithi110.stdout:2492: done (1 left) 2024-09-14T03:46:15.797 INFO:tasks.rados.rados.0.smithi110.stdout:2493: snap_create 2024-09-14T03:46:15.821 INFO:tasks.rados.rados.0.smithi110.stdout:2490: expect (ObjNum 759 snap 246 seq_num 759) 2024-09-14T03:46:16.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:15 smithi195 ceph-mon[82388]: Upgrade: Updating mds.b 2024-09-14T03:46:16.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:15 smithi195 ceph-mon[82388]: Deploying daemon mds.b on smithi195 2024-09-14T03:46:16.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:15 smithi195 ceph-mon[82388]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T03:46:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:15 smithi110 ceph-mon[88262]: Upgrade: Updating mds.b 2024-09-14T03:46:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:15 smithi110 ceph-mon[88262]: Deploying daemon mds.b on smithi195 2024-09-14T03:46:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:15 smithi110 ceph-mon[88262]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T03:46:16.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:15 smithi110 ceph-mon[86423]: Upgrade: Updating mds.b 2024-09-14T03:46:16.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:15 smithi110 ceph-mon[86423]: Deploying daemon mds.b on smithi195 2024-09-14T03:46:16.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:15 smithi110 ceph-mon[86423]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T03:46:16.620 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:16 smithi195 systemd[1]: Stopping Ceph mds.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:46:16.626 INFO:tasks.rados.rados.0.smithi110.stdout:2490: done (1 left) 2024-09-14T03:46:16.626 INFO:tasks.rados.rados.0.smithi110.stdout:2493: done (0 left) 2024-09-14T03:46:16.626 INFO:tasks.rados.rados.0.smithi110.stdout:2494: copy_from oid 48 from oid 32 current snap is 255 2024-09-14T03:46:16.626 INFO:tasks.rados.rados.0.smithi110.stdout:2495: rollback oid 19 current snap is 255 2024-09-14T03:46:16.626 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 247 2024-09-14T03:46:16.626 INFO:tasks.rados.rados.0.smithi110.stdout:2496: copy_from oid 36 from oid 37 current snap is 255 2024-09-14T03:46:16.626 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 255 2024-09-14T03:46:16.626 INFO:tasks.rados.rados.0.smithi110.stdout:2497: seq_num 781 ranges {15114240=688128,15802368=401408} 2024-09-14T03:46:16.710 INFO:tasks.rados.rados.0.smithi110.stdout:2497: writing smithi11075562-39 from 15114240 to 15802368 tid 1 2024-09-14T03:46:16.712 INFO:tasks.rados.rados.0.smithi110.stdout:2497: writing smithi11075562-39 from 15802368 to 16203776 tid 2 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:2498: snap_create 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:2496: finishing copy_from to smithi11075562-36 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:2496: got expected ENOENT (src dne) 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:2496: finishing copy_from racing read to smithi11075562-36 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:2495: finishing rollback tid 0 to smithi11075562-19 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:2495: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 1459 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:2494: finishing copy_from racing read to smithi11075562-48 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:2494: finishing copy_from to smithi11075562-48 2024-09-14T03:46:16.713 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1449 (ObjNum 760 snap 247 seq_num 760) dirty exists 2024-09-14T03:46:16.715 INFO:tasks.rados.rados.0.smithi110.stdout:2497: finishing write tid 1 to smithi11075562-39 2024-09-14T03:46:16.716 INFO:tasks.rados.rados.0.smithi110.stdout:2497: finishing write tid 2 to smithi11075562-39 2024-09-14T03:46:16.716 INFO:tasks.rados.rados.0.smithi110.stdout:2497: finishing write tid 3 to smithi11075562-39 2024-09-14T03:46:16.717 INFO:tasks.rados.rados.0.smithi110.stdout:2497: finishing write tid 4 to smithi11075562-39 2024-09-14T03:46:16.717 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1505 (ObjNum 781 snap 255 seq_num 781) dirty exists 2024-09-14T03:46:16.717 INFO:tasks.rados.rados.0.smithi110.stdout:2497: left oid 39 (ObjNum 781 snap 255 seq_num 781) 2024-09-14T03:46:16.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: pgmap v490: 121 pgs: 121 active+clean; 199 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-14T03:46:16.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T03:46:16.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:46:16.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:46:16.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[88262]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[86419]: 2024-09-14T03:46:16.618+0000 7fa26c043640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: pgmap v490: 121 pgs: 121 active+clean; 199 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:46:16.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:46:16.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:16 smithi110 ceph-mon[86423]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-14T03:46:16.958 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b[57021]: 2024-09-14T03:46:16.616+0000 7f3ca4506640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b[57021]: 2024-09-14T03:46:16.616+0000 7f3ca4506640 -1 mds.b *** got signal Terminated *** 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: pgmap v490: 121 pgs: 121 active+clean; 199 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:46:16.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:16 smithi195 ceph-mon[82388]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-14T03:46:17.633 INFO:tasks.rados.rados.0.smithi110.stdout:2494: done (4 left) 2024-09-14T03:46:17.633 INFO:tasks.rados.rados.0.smithi110.stdout:2495: done (3 left) 2024-09-14T03:46:17.633 INFO:tasks.rados.rados.0.smithi110.stdout:2496: done (2 left) 2024-09-14T03:46:17.633 INFO:tasks.rados.rados.0.smithi110.stdout:2497: done (1 left) 2024-09-14T03:46:17.633 INFO:tasks.rados.rados.0.smithi110.stdout:2498: done (0 left) 2024-09-14T03:46:17.634 INFO:tasks.rados.rados.0.smithi110.stdout:2499: read oid 28 snap -1 2024-09-14T03:46:17.634 INFO:tasks.rados.rados.0.smithi110.stdout:2499: expect (ObjNum 774 snap 251 seq_num 774) 2024-09-14T03:46:17.634 INFO:tasks.rados.rados.0.smithi110.stdout:2500: read oid 50 snap -1 2024-09-14T03:46:17.634 INFO:tasks.rados.rados.0.smithi110.stdout:2500: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:46:17.634 INFO:tasks.rados.rados.0.smithi110.stdout:2501: rmattr oid 39 current snap is 256 2024-09-14T03:46:17.634 INFO:tasks.rados.rados.0.smithi110.stdout:2502: setattr oid 19 current snap is 256 2024-09-14T03:46:17.636 INFO:tasks.rados.rados.0.smithi110.stdout:2503: read oid 14 snap -1 2024-09-14T03:46:17.636 INFO:tasks.rados.rados.0.smithi110.stdout:2503: expect (ObjNum 756 snap 244 seq_num 756) 2024-09-14T03:46:17.636 INFO:tasks.rados.rados.0.smithi110.stdout:2504: setattr oid 20 current snap is 256 2024-09-14T03:46:17.640 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1506 (ObjNum 781 snap 255 seq_num 781) dirty exists 2024-09-14T03:46:17.640 INFO:tasks.rados.rados.0.smithi110.stdout:2501: done (5 left) 2024-09-14T03:46:17.641 INFO:tasks.rados.rados.0.smithi110.stdout:2505: copy_from oid 48 from oid 34 current snap is 256 2024-09-14T03:46:17.641 INFO:tasks.rados.rados.0.smithi110.stdout:2506: setattr oid 44 current snap is 256 2024-09-14T03:46:17.641 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 1460 (ObjNum 33 snap 0 seq_num 66112944) dirty exists 2024-09-14T03:46:17.643 INFO:tasks.rados.rados.0.smithi110.stdout:2502: done (6 left) 2024-09-14T03:46:17.643 INFO:tasks.rados.rados.0.smithi110.stdout:2507: copy_from oid 5 from oid 9 current snap is 256 2024-09-14T03:46:17.644 INFO:tasks.rados.rados.0.smithi110.stdout:2508: read oid 27 snap -1 2024-09-14T03:46:17.644 INFO:tasks.rados.rados.0.smithi110.stdout:2508: expect (ObjNum 749 snap 243 seq_num 749) 2024-09-14T03:46:17.644 INFO:tasks.rados.rados.0.smithi110.stdout:2509: delete oid 37 current snap is 256 2024-09-14T03:46:17.644 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1577 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:46:17.648 INFO:tasks.rados.rados.0.smithi110.stdout:2507: finishing copy_from to smithi11075562-5 2024-09-14T03:46:17.648 INFO:tasks.rados.rados.0.smithi110.stdout:2507: got expected ENOENT (src dne) 2024-09-14T03:46:17.648 INFO:tasks.rados.rados.0.smithi110.stdout:2507: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:46:17.649 INFO:tasks.rados.rados.0.smithi110.stdout:2499: expect (ObjNum 774 snap 251 seq_num 774) 2024-09-14T03:46:17.689 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 413 (ObjNum 757 snap 244 seq_num 757) dirty exists 2024-09-14T03:46:17.689 INFO:tasks.rados.rados.0.smithi110.stdout:2508: expect (ObjNum 749 snap 243 seq_num 749) 2024-09-14T03:46:17.762 INFO:tasks.rados.rados.0.smithi110.stdout:2503: expect (ObjNum 756 snap 244 seq_num 756) 2024-09-14T03:46:17.986 INFO:tasks.rados.rados.0.smithi110.stdout:2505: finishing copy_from racing read to smithi11075562-48 2024-09-14T03:46:17.986 INFO:tasks.rados.rados.0.smithi110.stdout:2505: finishing copy_from to smithi11075562-48 2024-09-14T03:46:17.986 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1568 (ObjNum 701 snap 224 seq_num 701) dirty exists 2024-09-14T03:46:17.986 INFO:tasks.rados.rados.0.smithi110.stdout:2500: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:46:18.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[88262]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T03:46:18.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:reconnect 2024-09-14T03:46:18.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[88262]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-14T03:46:18.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:46:18.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[88262]: pgmap v494: 121 pgs: 121 active+clean; 202 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-14T03:46:18.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[86423]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T03:46:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:reconnect 2024-09-14T03:46:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[86423]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-14T03:46:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:46:18.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:18 smithi110 ceph-mon[86423]: pgmap v494: 121 pgs: 121 active+clean; 202 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-14T03:46:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:18 smithi195 ceph-mon[82388]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T03:46:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:18 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:reconnect 2024-09-14T03:46:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:18 smithi195 ceph-mon[82388]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-14T03:46:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:18 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:46:18.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:18 smithi195 ceph-mon[82388]: pgmap v494: 121 pgs: 121 active+clean; 202 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-14T03:46:19.421 INFO:tasks.rados.rados.0.smithi110.stdout:2499: done (8 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2500: done (7 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2503: done (6 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2504: done (5 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2505: done (4 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2506: done (3 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2507: done (2 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2508: done (1 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2509: done (0 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2510: rmattr oid 41 current snap is 256 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2510: done (0 left) 2024-09-14T03:46:19.422 INFO:tasks.rados.rados.0.smithi110.stdout:2511: delete oid 3 current snap is 256 2024-09-14T03:46:19.423 INFO:tasks.rados.rados.0.smithi110.stdout:2511: done (0 left) 2024-09-14T03:46:19.424 INFO:tasks.rados.rados.0.smithi110.stdout:2512: rollback oid 38 current snap is 256 2024-09-14T03:46:19.424 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 239 2024-09-14T03:46:19.424 INFO:tasks.rados.rados.0.smithi110.stdout:2513: rollback oid 39 current snap is 256 2024-09-14T03:46:19.424 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 231 2024-09-14T03:46:19.424 INFO:tasks.rados.rados.0.smithi110.stdout:2514: read oid 2 snap 255 2024-09-14T03:46:19.424 INFO:tasks.rados.rados.0.smithi110.stdout:2514: expect deleted 2024-09-14T03:46:19.424 INFO:tasks.rados.rados.0.smithi110.stdout:2515: read oid 37 snap 247 2024-09-14T03:46:19.424 INFO:tasks.rados.rados.0.smithi110.stdout:2515: expect deleted 2024-09-14T03:46:19.425 INFO:tasks.rados.rados.0.smithi110.stdout:2516: rollback oid 4 current snap is 256 2024-09-14T03:46:19.425 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 4 to 255 2024-09-14T03:46:19.425 INFO:tasks.rados.rados.0.smithi110.stdout:2517: rollback oid 50 current snap is 256 2024-09-14T03:46:19.425 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 253 2024-09-14T03:46:19.425 INFO:tasks.rados.rados.0.smithi110.stdout:2518: snap_create 2024-09-14T03:46:19.425 INFO:tasks.rados.rados.0.smithi110.stdout:2512: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:46:19.427 INFO:tasks.rados.rados.0.smithi110.stdout:2512: finishing rollback tid 2 to smithi11075562-38 2024-09-14T03:46:19.427 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1582 (ObjNum 718 snap 229 seq_num 718) dirty exists 2024-09-14T03:46:19.427 INFO:tasks.rados.rados.0.smithi110.stdout:2516: finishing rollback tid 0 to smithi11075562-4 2024-09-14T03:46:19.428 INFO:tasks.rados.rados.0.smithi110.stdout:2516: finishing rollback tid 1 to smithi11075562-4 2024-09-14T03:46:19.428 INFO:tasks.rados.rados.0.smithi110.stdout:2516: finishing rollback tid 2 to smithi11075562-4 2024-09-14T03:46:19.429 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1571 (ObjNum 753 snap 244 seq_num 753) dirty exists 2024-09-14T03:46:19.430 INFO:tasks.rados.rados.0.smithi110.stdout:2513: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:46:19.430 INFO:tasks.rados.rados.0.smithi110.stdout:2517: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:46:19.431 INFO:tasks.rados.rados.0.smithi110.stdout:2513: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:46:19.431 INFO:tasks.rados.rados.0.smithi110.stdout:2517: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:46:19.431 INFO:tasks.rados.rados.0.smithi110.stdout:2517: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:46:19.431 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 1494 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:46:19.431 INFO:tasks.rados.rados.0.smithi110.stdout:2513: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:46:19.431 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1509 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:46:19.750 INFO:tasks.rados.rados.0.smithi110.stdout:2512: done (6 left) 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:2513: done (5 left) 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:2514: done (4 left) 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:2515: done (3 left) 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:2516: done (2 left) 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:2517: done (1 left) 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:2518: done (0 left) 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:2519: copy_from oid 12 from oid 43 current snap is 257 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 257 2024-09-14T03:46:19.751 INFO:tasks.rados.rados.0.smithi110.stdout:2520: seq_num 784 ranges {3178496=737280,3915776=401408} 2024-09-14T03:46:19.771 INFO:tasks.rados.rados.0.smithi110.stdout:2520: writing smithi11075562-49 from 3178496 to 3915776 tid 1 2024-09-14T03:46:19.774 INFO:tasks.rados.rados.0.smithi110.stdout:2520: writing smithi11075562-49 from 3915776 to 4317184 tid 2 2024-09-14T03:46:19.774 INFO:tasks.rados.rados.0.smithi110.stdout:2521: copy_from oid 6 from oid 21 current snap is 257 2024-09-14T03:46:19.774 INFO:tasks.rados.rados.0.smithi110.stdout:2522: snap_remove snap 243 2024-09-14T03:46:19.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:19 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:rejoin 2024-09-14T03:46:19.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:19 smithi110 ceph-mon[88262]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-14T03:46:19.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:19 smithi110 ceph-mon[88262]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:46:19.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:19 smithi110 ceph-mon[88262]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T03:46:19.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:19 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:rejoin 2024-09-14T03:46:19.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:19 smithi110 ceph-mon[86423]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-14T03:46:19.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:19 smithi110 ceph-mon[86423]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:46:19.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:19 smithi110 ceph-mon[86423]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T03:46:19.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:19 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:rejoin 2024-09-14T03:46:19.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:19 smithi195 ceph-mon[82388]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-14T03:46:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:19 smithi195 ceph-mon[82388]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:46:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:19 smithi195 ceph-mon[82388]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T03:46:20.644 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:46:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:46:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:46:20.757 INFO:tasks.rados.rados.0.smithi110.stdout:2522: done (3 left) 2024-09-14T03:46:20.757 INFO:tasks.rados.rados.0.smithi110.stdout:2523: snap_remove snap 249 2024-09-14T03:46:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[86423]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:46:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:46:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:active 2024-09-14T03:46:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[86423]: fsmap cephfs:1 {0=a=up:active} 2024-09-14T03:46:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[86423]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T03:46:20.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[86423]: pgmap v497: 121 pgs: 121 active+clean; 208 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.9 MiB/s wr, 33 op/s 2024-09-14T03:46:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[88262]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:46:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:46:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:active 2024-09-14T03:46:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[88262]: fsmap cephfs:1 {0=a=up:active} 2024-09-14T03:46:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[88262]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T03:46:20.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:20 smithi110 ceph-mon[88262]: pgmap v497: 121 pgs: 121 active+clean; 208 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.9 MiB/s wr, 33 op/s 2024-09-14T03:46:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:20 smithi195 ceph-mon[82388]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:46:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:20 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:46:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:20 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.110:6834/1777878812,v1:172.21.15.110:6835/1777878812] up:active 2024-09-14T03:46:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:20 smithi195 ceph-mon[82388]: fsmap cephfs:1 {0=a=up:active} 2024-09-14T03:46:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:20 smithi195 ceph-mon[82388]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T03:46:20.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:20 smithi195 ceph-mon[82388]: pgmap v497: 121 pgs: 121 active+clean; 208 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.9 MiB/s wr, 33 op/s 2024-09-14T03:46:21.763 INFO:tasks.rados.rados.0.smithi110.stdout:2523: done (3 left) 2024-09-14T03:46:21.763 INFO:tasks.rados.rados.0.smithi110.stdout:2524: read oid 20 snap 255 2024-09-14T03:46:21.763 INFO:tasks.rados.rados.0.smithi110.stdout:2524: expect deleted 2024-09-14T03:46:21.763 INFO:tasks.rados.rados.0.smithi110.stdout:2525: snap_remove snap 247 2024-09-14T03:46:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:21 smithi110 ceph-mon[86423]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T03:46:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:21 smithi110 ceph-mon[88262]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T03:46:21.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:46:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:46:21] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-14T03:46:21.947 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:46:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[111676]: 2024-09-14T03:46:21.766+0000 7fca01337640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-14T03:46:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:21 smithi195 ceph-mon[82388]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T03:46:22.768 INFO:tasks.rados.rados.0.smithi110.stdout:2525: done (4 left) 2024-09-14T03:46:22.768 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 257 2024-09-14T03:46:22.768 INFO:tasks.rados.rados.0.smithi110.stdout:2526: seq_num 785 ranges {0=753664,753664=401408} 2024-09-14T03:46:22.773 INFO:tasks.rados.rados.0.smithi110.stdout:2526: writing smithi11075562-15 from 0 to 753664 tid 1 2024-09-14T03:46:22.776 INFO:tasks.rados.rados.0.smithi110.stdout:2526: writing smithi11075562-15 from 753664 to 1155072 tid 2 2024-09-14T03:46:22.776 INFO:tasks.rados.rados.0.smithi110.stdout:2527: rollback oid 16 current snap is 257 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 256 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2528: read oid 35 snap -1 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2528: expect deleted 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2529: snap_create 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2521: finishing copy_from to smithi11075562-6 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2521: got expected ENOENT (src dne) 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2521: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2519: finishing copy_from racing read to smithi11075562-12 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2520: finishing write tid 1 to smithi11075562-49 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2520: finishing write tid 2 to smithi11075562-49 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2520: finishing write tid 3 to smithi11075562-49 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2519: finishing copy_from to smithi11075562-12 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1497 (ObjNum 751 snap 244 seq_num 751) dirty exists 2024-09-14T03:46:22.777 INFO:tasks.rados.rados.0.smithi110.stdout:2520: finishing write tid 4 to smithi11075562-49 2024-09-14T03:46:22.778 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1479 (ObjNum 784 snap 257 seq_num 784) dirty exists 2024-09-14T03:46:22.778 INFO:tasks.rados.rados.0.smithi110.stdout:2520: left oid 49 (ObjNum 784 snap 257 seq_num 784) 2024-09-14T03:46:22.778 INFO:tasks.rados.rados.0.smithi110.stdout:2527: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:46:22.778 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 0 (ObjNum 161 snap 0 seq_num 64674896) dirty dne 2024-09-14T03:46:22.783 INFO:tasks.rados.rados.0.smithi110.stdout:2526: finishing write tid 1 to smithi11075562-15 2024-09-14T03:46:22.784 INFO:tasks.rados.rados.0.smithi110.stdout:2526: finishing write tid 2 to smithi11075562-15 2024-09-14T03:46:22.784 INFO:tasks.rados.rados.0.smithi110.stdout:2526: finishing write tid 3 to smithi11075562-15 2024-09-14T03:46:22.785 INFO:tasks.rados.rados.0.smithi110.stdout:2526: finishing write tid 4 to smithi11075562-15 2024-09-14T03:46:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1495 (ObjNum 785 snap 257 seq_num 785) dirty exists 2024-09-14T03:46:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:2526: left oid 15 (ObjNum 785 snap 257 seq_num 785) 2024-09-14T03:46:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:22 smithi110 ceph-mon[88262]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T03:46:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:22 smithi110 ceph-mon[88262]: pgmap v500: 121 pgs: 121 active+clean; 214 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 31 op/s 2024-09-14T03:46:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:22 smithi110 ceph-mon[86423]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T03:46:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:22 smithi110 ceph-mon[86423]: pgmap v500: 121 pgs: 121 active+clean; 214 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 31 op/s 2024-09-14T03:46:23.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:22 smithi195 ceph-mon[82388]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T03:46:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:22 smithi195 ceph-mon[82388]: pgmap v500: 121 pgs: 121 active+clean; 214 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 31 op/s 2024-09-14T03:46:23.753 INFO:tasks.rados.rados.0.smithi110.stdout:2519: done (7 left) 2024-09-14T03:46:23.753 INFO:tasks.rados.rados.0.smithi110.stdout:2520: done (6 left) 2024-09-14T03:46:23.753 INFO:tasks.rados.rados.0.smithi110.stdout:2521: done (5 left) 2024-09-14T03:46:23.753 INFO:tasks.rados.rados.0.smithi110.stdout:2524: done (4 left) 2024-09-14T03:46:23.753 INFO:tasks.rados.rados.0.smithi110.stdout:2526: done (3 left) 2024-09-14T03:46:23.753 INFO:tasks.rados.rados.0.smithi110.stdout:2527: done (2 left) 2024-09-14T03:46:23.753 INFO:tasks.rados.rados.0.smithi110.stdout:2528: done (1 left) 2024-09-14T03:46:23.754 INFO:tasks.rados.rados.0.smithi110.stdout:2529: done (0 left) 2024-09-14T03:46:23.754 INFO:tasks.rados.rados.0.smithi110.stdout:2530: delete oid 35 current snap is 258 2024-09-14T03:46:23.757 INFO:tasks.rados.rados.0.smithi110.stdout:2530: done (0 left) 2024-09-14T03:46:23.757 INFO:tasks.rados.rados.0.smithi110.stdout:2531: snap_create 2024-09-14T03:46:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:23 smithi110 ceph-mon[88262]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T03:46:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:23 smithi110 ceph-mon[88262]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T03:46:24.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:23 smithi110 ceph-mon[86423]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T03:46:24.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:23 smithi110 ceph-mon[86423]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T03:46:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:23 smithi195 ceph-mon[82388]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T03:46:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:23 smithi195 ceph-mon[82388]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T03:46:24.753 INFO:tasks.rados.rados.0.smithi110.stdout:2531: done (0 left) 2024-09-14T03:46:24.754 INFO:tasks.rados.rados.0.smithi110.stdout:2532: copy_from oid 2 from oid 45 current snap is 259 2024-09-14T03:46:24.754 INFO:tasks.rados.rados.0.smithi110.stdout:2533: rmattr oid 23 current snap is 259 2024-09-14T03:46:24.754 INFO:tasks.rados.rados.0.smithi110.stdout:2534: rmattr oid 38 current snap is 259 2024-09-14T03:46:24.754 INFO:tasks.rados.rados.0.smithi110.stdout:2534: done (2 left) 2024-09-14T03:46:24.754 INFO:tasks.rados.rados.0.smithi110.stdout:2535: delete oid 48 current snap is 259 2024-09-14T03:46:24.759 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1495 (ObjNum 33 snap 0 seq_num 64461360) dirty exists 2024-09-14T03:46:24.761 INFO:tasks.rados.rados.0.smithi110.stdout:2533: done (2 left) 2024-09-14T03:46:24.761 INFO:tasks.rados.rados.0.smithi110.stdout:2535: done (1 left) 2024-09-14T03:46:24.761 INFO:tasks.rados.rados.0.smithi110.stdout:2536: snap_remove snap 231 2024-09-14T03:46:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:24 smithi195 ceph-mon[82388]: pgmap v503: 121 pgs: 121 active+clean; 216 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-14T03:46:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:24 smithi195 ceph-mon[82388]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T03:46:25.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:24 smithi110 ceph-mon[86423]: pgmap v503: 121 pgs: 121 active+clean; 216 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-14T03:46:25.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:24 smithi110 ceph-mon[86423]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T03:46:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:24 smithi110 ceph-mon[88262]: pgmap v503: 121 pgs: 121 active+clean; 216 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-14T03:46:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:24 smithi110 ceph-mon[88262]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T03:46:25.756 INFO:tasks.rados.rados.0.smithi110.stdout:2536: done (1 left) 2024-09-14T03:46:25.757 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 259 2024-09-14T03:46:25.757 INFO:tasks.rados.rados.0.smithi110.stdout:2537: seq_num 788 ranges {0=802816,802816=401408} 2024-09-14T03:46:25.762 INFO:tasks.rados.rados.0.smithi110.stdout:2537: writing smithi11075562-22 from 0 to 802816 tid 1 2024-09-14T03:46:25.765 INFO:tasks.rados.rados.0.smithi110.stdout:2537: writing smithi11075562-22 from 802816 to 1204224 tid 2 2024-09-14T03:46:25.765 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 259 2024-09-14T03:46:25.765 INFO:tasks.rados.rados.0.smithi110.stdout:2532: finishing copy_from racing read to smithi11075562-2 2024-09-14T03:46:25.766 INFO:tasks.rados.rados.0.smithi110.stdout:2532: finishing copy_from to smithi11075562-2 2024-09-14T03:46:25.766 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 1643 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:46:25.766 INFO:tasks.rados.rados.0.smithi110.stdout:2538: seq_num 789 ranges {14180352=417792,14598144=663552,15261696=139264} 2024-09-14T03:46:25.844 INFO:tasks.rados.rados.0.smithi110.stdout:2538: writing smithi11075562-39 from 14180352 to 14598144 tid 1 2024-09-14T03:46:25.847 INFO:tasks.rados.rados.0.smithi110.stdout:2538: writing smithi11075562-39 from 14598144 to 15261696 tid 2 2024-09-14T03:46:25.848 INFO:tasks.rados.rados.0.smithi110.stdout:2538: writing smithi11075562-39 from 15261696 to 15400960 tid 3 2024-09-14T03:46:25.848 INFO:tasks.rados.rados.0.smithi110.stdout:2532: done (2 left) 2024-09-14T03:46:25.848 INFO:tasks.rados.rados.0.smithi110.stdout:2539: snap_remove snap 253 2024-09-14T03:46:26.762 INFO:tasks.rados.rados.0.smithi110.stdout:2537: finishing write tid 1 to smithi11075562-22 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2537: finishing write tid 2 to smithi11075562-22 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2537: finishing write tid 3 to smithi11075562-22 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2537: finishing write tid 4 to smithi11075562-22 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1504 (ObjNum 788 snap 259 seq_num 788) dirty exists 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2537: left oid 22 (ObjNum 788 snap 259 seq_num 788) 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2538: finishing write tid 1 to smithi11075562-39 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2538: finishing write tid 2 to smithi11075562-39 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2538: finishing write tid 3 to smithi11075562-39 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2538: finishing write tid 4 to smithi11075562-39 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2538: finishing write tid 5 to smithi11075562-39 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1513 (ObjNum 789 snap 259 seq_num 789) dirty exists 2024-09-14T03:46:26.763 INFO:tasks.rados.rados.0.smithi110.stdout:2538: left oid 39 (ObjNum 789 snap 259 seq_num 789) 2024-09-14T03:46:26.764 INFO:tasks.rados.rados.0.smithi110.stdout:2537: done (2 left) 2024-09-14T03:46:26.764 INFO:tasks.rados.rados.0.smithi110.stdout:2538: done (1 left) 2024-09-14T03:46:26.764 INFO:tasks.rados.rados.0.smithi110.stdout:2539: done (0 left) 2024-09-14T03:46:26.764 INFO:tasks.rados.rados.0.smithi110.stdout:2540: setattr oid 27 current snap is 259 2024-09-14T03:46:26.764 INFO:tasks.rados.rados.0.smithi110.stdout:2541: rmattr oid 39 current snap is 259 2024-09-14T03:46:26.764 INFO:tasks.rados.rados.0.smithi110.stdout:2542: rollback oid 7 current snap is 259 2024-09-14T03:46:26.765 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 7 to 250 2024-09-14T03:46:26.765 INFO:tasks.rados.rados.0.smithi110.stdout:2543: snap_remove snap 254 2024-09-14T03:46:26.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:26 smithi195 ceph-mon[82388]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T03:46:26.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:26 smithi195 ceph-mon[82388]: pgmap v506: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 217 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 282 KiB/s wr, 14 op/s 2024-09-14T03:46:26.959 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:26 smithi195 bash[105045]: time="2024-09-14T03:46:26Z" level=warning msg="StopSignal SIGTERM failed to stop container ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b in 10 seconds, resorting to SIGKILL" 2024-09-14T03:46:26.959 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:26 smithi195 podman[105045]: 2024-09-14 03:46:26.635955163 +0000 UTC m=+10.131572120 container died d306b18ee20b211b6443f08ba70f6cd57bdc8a9c73367fb3fe379c093680ea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:46:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:26 smithi110 ceph-mon[86423]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T03:46:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:26 smithi110 ceph-mon[86423]: pgmap v506: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 217 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 282 KiB/s wr, 14 op/s 2024-09-14T03:46:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:26 smithi110 ceph-mon[88262]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T03:46:27.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:26 smithi110 ceph-mon[88262]: pgmap v506: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 217 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 282 KiB/s wr, 14 op/s 2024-09-14T03:46:27.459 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:27 smithi195 podman[105045]: 2024-09-14 03:46:27.217329999 +0000 UTC m=+10.712946993 container cleanup d306b18ee20b211b6443f08ba70f6cd57bdc8a9c73367fb3fe379c093680ea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T03:46:27.459 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:27 smithi195 bash[105045]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b 2024-09-14T03:46:27.758 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:27 smithi195 podman[105059]: 2024-09-14 03:46:27.468538986 +0000 UTC m=+0.833626421 container remove d306b18ee20b211b6443f08ba70f6cd57bdc8a9c73367fb3fe379c093680ea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-eb16b97, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:46:27.758 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:27 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.b.service: Main process exited, code=exited, status=137/n/a 2024-09-14T03:46:27.758 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:27 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.b.service: Failed with result 'exit-code'. 2024-09-14T03:46:27.758 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:27 smithi195 systemd[1]: Stopped Ceph mds.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:46:27.758 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:27 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.b.service: Consumed 1.154s CPU time. 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:2542: finishing rollback tid 0 to smithi11075562-7 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:2542: finishing rollback tid 1 to smithi11075562-7 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1571 (ObjNum 749 snap 243 seq_num 749) dirty exists 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1514 (ObjNum 789 snap 259 seq_num 789) dirty exists 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:2542: finishing rollback tid 2 to smithi11075562-7 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1585 (ObjNum 738 snap 235 seq_num 738) dirty exists 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:2540: done (3 left) 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:2541: done (2 left) 2024-09-14T03:46:27.766 INFO:tasks.rados.rados.0.smithi110.stdout:2542: done (1 left) 2024-09-14T03:46:27.767 INFO:tasks.rados.rados.0.smithi110.stdout:2543: done (0 left) 2024-09-14T03:46:27.767 INFO:tasks.rados.rados.0.smithi110.stdout:2544: delete oid 23 current snap is 259 2024-09-14T03:46:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:2544: done (0 left) 2024-09-14T03:46:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:2545: read oid 50 snap -1 2024-09-14T03:46:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:2545: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:46:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:2546: rollback oid 41 current snap is 259 2024-09-14T03:46:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 239 2024-09-14T03:46:27.773 INFO:tasks.rados.rados.0.smithi110.stdout:2547: delete oid 48 current snap is 259 2024-09-14T03:46:27.779 INFO:tasks.rados.rados.0.smithi110.stdout:2547: done (2 left) 2024-09-14T03:46:27.779 INFO:tasks.rados.rados.0.smithi110.stdout:2548: delete oid 20 current snap is 259 2024-09-14T03:46:27.779 INFO:tasks.rados.rados.0.smithi110.stdout:2546: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:46:27.780 INFO:tasks.rados.rados.0.smithi110.stdout:2546: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:46:27.780 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1516 (ObjNum 734 snap 233 seq_num 734) dirty exists 2024-09-14T03:46:27.782 INFO:tasks.rados.rados.0.smithi110.stdout:2546: done (2 left) 2024-09-14T03:46:27.782 INFO:tasks.rados.rados.0.smithi110.stdout:2548: done (1 left) 2024-09-14T03:46:27.782 INFO:tasks.rados.rados.0.smithi110.stdout:2549: snap_create 2024-09-14T03:46:27.824 INFO:tasks.rados.rados.0.smithi110.stdout:2545: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:46:28.026 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:27 smithi195 systemd[1]: Starting Ceph mds.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:46:28.027 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:27 smithi195 ceph-mon[82388]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T03:46:28.027 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:27 smithi110 ceph-mon[88262]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T03:46:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:27 smithi110 ceph-mon[86423]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T03:46:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:28.284 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:28 smithi195 podman[105188]: 2024-09-14 03:46:27.92833285 +0000 UTC m=+0.015667304 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:46:28.284 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:28 smithi195 podman[105188]: 2024-09-14 03:46:28.028081557 +0000 UTC m=+0.115415999 container create 08047cadae86b419420482ec73ad1e4951e5a2f3a5968d7b5930a07d1b8b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-14T03:46:28.284 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:28 smithi195 podman[105188]: 2024-09-14 03:46:28.256116789 +0000 UTC m=+0.343451230 container init 08047cadae86b419420482ec73ad1e4951e5a2f3a5968d7b5930a07d1b8b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-14T03:46:28.285 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:28 smithi195 podman[105188]: 2024-09-14 03:46:28.259119274 +0000 UTC m=+0.346453716 container start 08047cadae86b419420482ec73ad1e4951e5a2f3a5968d7b5930a07d1b8b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True) 2024-09-14T03:46:28.708 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:28 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b[105203]: starting mds.b at 2024-09-14T03:46:28.709 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:28 smithi195 bash[105188]: 08047cadae86b419420482ec73ad1e4951e5a2f3a5968d7b5930a07d1b8b4061 2024-09-14T03:46:28.709 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:46:28 smithi195 systemd[1]: Started Ceph mds.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:46:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:28 smithi195 ceph-mon[82388]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T03:46:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:28 smithi195 ceph-mon[82388]: pgmap v509: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 219 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 580 KiB/s wr, 11 op/s 2024-09-14T03:46:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:28 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:28 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:28 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:28 smithi195 ceph-mon[82388]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T03:46:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[88262]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T03:46:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[88262]: pgmap v509: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 219 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 580 KiB/s wr, 11 op/s 2024-09-14T03:46:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[88262]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T03:46:29.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[86423]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T03:46:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[86423]: pgmap v509: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 219 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 580 KiB/s wr, 11 op/s 2024-09-14T03:46:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:28 smithi110 ceph-mon[86423]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T03:46:29.257 INFO:tasks.rados.rados.0.smithi110.stdout:2545: done (1 left) 2024-09-14T03:46:29.258 INFO:tasks.rados.rados.0.smithi110.stdout:2549: done (0 left) 2024-09-14T03:46:29.258 INFO:tasks.rados.rados.0.smithi110.stdout:2550: snap_remove snap 248 2024-09-14T03:46:29.775 INFO:tasks.rados.rados.0.smithi110.stdout:2550: done (0 left) 2024-09-14T03:46:29.775 INFO:tasks.rados.rados.0.smithi110.stdout:2551: snap_create 2024-09-14T03:46:30.016 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:29 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.195:6832/2567437763,v1:172.21.15.195:6833/2567437763] up:boot 2024-09-14T03:46:30.017 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:29 smithi195 ceph-mon[82388]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:46:30.017 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:29 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:46:30.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:29 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.195:6832/2567437763,v1:172.21.15.195:6833/2567437763] up:boot 2024-09-14T03:46:30.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:29 smithi110 ceph-mon[88262]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:46:30.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:29 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:46:30.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:29 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.195:6832/2567437763,v1:172.21.15.195:6833/2567437763] up:boot 2024-09-14T03:46:30.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:29 smithi110 ceph-mon[86423]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:46:30.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:29 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:46:30.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:46:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:46:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:46:30.780 INFO:tasks.rados.rados.0.smithi110.stdout:2551: done (0 left) 2024-09-14T03:46:30.780 INFO:tasks.rados.rados.0.smithi110.stdout:2552: snap_remove snap 229 2024-09-14T03:46:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:30 smithi195 ceph-mon[82388]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T03:46:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:30 smithi195 ceph-mon[82388]: pgmap v512: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-14T03:46:31.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:30 smithi110 ceph-mon[88262]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T03:46:31.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:30 smithi110 ceph-mon[88262]: pgmap v512: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-14T03:46:31.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:30 smithi110 ceph-mon[86423]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T03:46:31.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:30 smithi110 ceph-mon[86423]: pgmap v512: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-14T03:46:31.781 INFO:tasks.rados.rados.0.smithi110.stdout:2552: done (0 left) 2024-09-14T03:46:31.782 INFO:tasks.rados.rados.0.smithi110.stdout:2553: setattr oid 43 current snap is 261 2024-09-14T03:46:31.785 INFO:tasks.rados.rados.0.smithi110.stdout:2554: setattr oid 29 current snap is 261 2024-09-14T03:46:31.789 INFO:tasks.rados.rados.0.smithi110.stdout:2555: rmattr oid 12 current snap is 261 2024-09-14T03:46:31.790 INFO:tasks.rados.rados.0.smithi110.stdout:2555: done (2 left) 2024-09-14T03:46:31.790 INFO:tasks.rados.rados.0.smithi110.stdout:2556: read oid 3 snap -1 2024-09-14T03:46:31.790 INFO:tasks.rados.rados.0.smithi110.stdout:2556: expect deleted 2024-09-14T03:46:31.790 INFO:tasks.rados.rados.0.smithi110.stdout:2557: copy_from oid 15 from oid 24 current snap is 261 2024-09-14T03:46:31.790 INFO:tasks.rados.rados.0.smithi110.stdout:2558: read oid 32 snap 246 2024-09-14T03:46:31.790 INFO:tasks.rados.rados.0.smithi110.stdout:2558: expect (ObjNum 736 snap 235 seq_num 736) 2024-09-14T03:46:31.790 INFO:tasks.rados.rados.0.smithi110.stdout:2559: setattr oid 26 current snap is 261 2024-09-14T03:46:31.793 INFO:tasks.rados.rados.0.smithi110.stdout:2560: delete oid 30 current snap is 261 2024-09-14T03:46:31.799 INFO:tasks.rados.rados.0.smithi110.stdout:2557: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:46:31.805 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 789 (ObjNum 145 snap 0 seq_num 1496888760) dirty exists 2024-09-14T03:46:31.807 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1586 (ObjNum 653 snap 202 seq_num 653) dirty exists 2024-09-14T03:46:31.810 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1517 (ObjNum 751 snap 244 seq_num 751) dirty exists 2024-09-14T03:46:31.810 INFO:tasks.rados.rados.0.smithi110.stdout:2557: finishing copy_from to smithi11075562-15 2024-09-14T03:46:31.810 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1493 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:46:31.812 INFO:tasks.rados.rados.0.smithi110.stdout:2558: expect (ObjNum 736 snap 235 seq_num 736) 2024-09-14T03:46:31.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:46:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:46:31] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.51.0" 2024-09-14T03:46:31.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:31 smithi110 ceph-mon[88262]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T03:46:31.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:31 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:31.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:31 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:31.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:31 smithi110 ceph-mon[86423]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T03:46:31.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:31 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:31.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:31 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:31.968 INFO:tasks.rados.rados.0.smithi110.stdout:2553: done (6 left) 2024-09-14T03:46:31.968 INFO:tasks.rados.rados.0.smithi110.stdout:2554: done (5 left) 2024-09-14T03:46:31.969 INFO:tasks.rados.rados.0.smithi110.stdout:2556: done (4 left) 2024-09-14T03:46:31.969 INFO:tasks.rados.rados.0.smithi110.stdout:2557: done (3 left) 2024-09-14T03:46:31.969 INFO:tasks.rados.rados.0.smithi110.stdout:2558: done (2 left) 2024-09-14T03:46:31.969 INFO:tasks.rados.rados.0.smithi110.stdout:2559: done (1 left) 2024-09-14T03:46:31.969 INFO:tasks.rados.rados.0.smithi110.stdout:2560: done (0 left) 2024-09-14T03:46:31.969 INFO:tasks.rados.rados.0.smithi110.stdout:2561: snap_remove snap 245 2024-09-14T03:46:32.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:31 smithi195 ceph-mon[82388]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T03:46:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:31 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:31 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:32.795 INFO:tasks.rados.rados.0.smithi110.stdout:2561: done (0 left) 2024-09-14T03:46:32.795 INFO:tasks.rados.rados.0.smithi110.stdout:2562: rollback oid 23 current snap is 261 2024-09-14T03:46:32.795 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 23 to 259 2024-09-14T03:46:32.795 INFO:tasks.rados.rados.0.smithi110.stdout:2563: read oid 40 snap -1 2024-09-14T03:46:32.795 INFO:tasks.rados.rados.0.smithi110.stdout:2563: expect (ObjNum 745 snap 242 seq_num 745) 2024-09-14T03:46:32.795 INFO:tasks.rados.rados.0.smithi110.stdout:2564: snap_create 2024-09-14T03:46:32.811 INFO:tasks.rados.rados.0.smithi110.stdout:2562: finishing rollback tid 1 to smithi11075562-23 2024-09-14T03:46:32.811 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1496 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:46:32.820 INFO:tasks.rados.rados.0.smithi110.stdout:2563: expect (ObjNum 745 snap 242 seq_num 745) 2024-09-14T03:46:32.933 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:32 smithi195 ceph-mon[82388]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T03:46:32.933 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:32 smithi195 ceph-mon[82388]: pgmap v515: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 832 KiB/s wr, 8 op/s 2024-09-14T03:46:32.933 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:32 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:32.933 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:32 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:32.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:32 smithi110 ceph-mon[88262]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T03:46:32.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:32 smithi110 ceph-mon[88262]: pgmap v515: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 832 KiB/s wr, 8 op/s 2024-09-14T03:46:32.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:32 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:32.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:32 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:32.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:32 smithi110 ceph-mon[86423]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T03:46:32.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:32 smithi110 ceph-mon[86423]: pgmap v515: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 222 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 832 KiB/s wr, 8 op/s 2024-09-14T03:46:32.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:32 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:32.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:32 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:33.062 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:46:33.624 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:46:33.624 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (7m) 24s ago 11m 19.7M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:46:33.624 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (6m) 1s ago 11m 81.9M - 9.4.7 954c08fa6188 03236d3bc210 2024-09-14T03:46:33.624 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (27s) 24s ago 12m 13.2M - 19.3.0-4907-ga2782315 8801d2529e43 ddd26c7ac6c2 2024-09-14T03:46:33.624 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (5s) 1s ago 12m 14.1M - 19.3.0-4907-ga2782315 8801d2529e43 08047cadae86 2024-09-14T03:46:33.624 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (6m) 1s ago 16m 455M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:46:33.624 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (6m) 24s ago 17m 530M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (6m) 24s ago 17m 68.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (5m) 1s ago 16m 53.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (5m) 24s ago 16m 58.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (7m) 24s ago 11m 10.1M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (6m) 1s ago 10m 11.9M - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (4m) 24s ago 15m 274M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (4m) 24s ago 15m 182M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (3m) 24s ago 15m 189M 2480M 19.3.0-4907-ga2782315 8801d2529e43 0cdc58c38711 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (3m) 24s ago 14m 139M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2283ebcb5f10 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (2m) 1s ago 14m 231M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6921b0a30047 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (114s) 1s ago 13m 191M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8fe7f08d5882 2024-09-14T03:46:33.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (80s) 1s ago 13m 121M 2480M 19.3.0-4907-ga2782315 8801d2529e43 5305ca81bc26 2024-09-14T03:46:33.626 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (46s) 1s ago 13m 121M 2480M 19.3.0-4907-ga2782315 8801d2529e43 98e6e5252ce4 2024-09-14T03:46:33.626 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (6m) 1s ago 11m 54.8M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:46:33.753 INFO:tasks.rados.rados.0.smithi110.stdout:2562: done (2 left) 2024-09-14T03:46:33.753 INFO:tasks.rados.rados.0.smithi110.stdout:2563: done (1 left) 2024-09-14T03:46:33.753 INFO:tasks.rados.rados.0.smithi110.stdout:2564: done (0 left) 2024-09-14T03:46:33.753 INFO:tasks.rados.rados.0.smithi110.stdout:2565: read oid 10 snap -1 2024-09-14T03:46:33.753 INFO:tasks.rados.rados.0.smithi110.stdout:2565: expect (ObjNum 754 snap 244 seq_num 754) 2024-09-14T03:46:33.753 INFO:tasks.rados.rados.0.smithi110.stdout:2566: snap_create 2024-09-14T03:46:33.758 INFO:tasks.rados.rados.0.smithi110.stdout:2565: expect (ObjNum 754 snap 244 seq_num 754) 2024-09-14T03:46:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:33 smithi110 ceph-mon[88262]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T03:46:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:33 smithi110 ceph-mon[88262]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T03:46:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:33 smithi110 ceph-mon[86423]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T03:46:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:33 smithi110 ceph-mon[86423]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T03:46:33.966 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:46:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 15 2024-09-14T03:46:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:46:33.968 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:46:34.095 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:33 smithi195 ceph-mon[82388]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T03:46:34.096 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:33 smithi195 ceph-mon[82388]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "in_progress": true, 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "services_complete": [ 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr", 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "mon", 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "mds", 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "osd" 2024-09-14T03:46:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2024-09-14T03:46:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "progress": "15/20 daemons upgraded", 2024-09-14T03:46:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "message": "Currently upgrading mds daemons", 2024-09-14T03:46:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "is_paused": false 2024-09-14T03:46:34.250 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:46:34.755 INFO:tasks.rados.rados.0.smithi110.stdout:2565: done (1 left) 2024-09-14T03:46:34.755 INFO:tasks.rados.rados.0.smithi110.stdout:2566: done (0 left) 2024-09-14T03:46:34.755 INFO:tasks.rados.rados.0.smithi110.stdout:2567: read oid 28 snap 252 2024-09-14T03:46:34.755 INFO:tasks.rados.rados.0.smithi110.stdout:2567: expect (ObjNum 774 snap 251 seq_num 774) 2024-09-14T03:46:34.755 INFO:tasks.rados.rados.0.smithi110.stdout:2568: snap_remove snap 255 2024-09-14T03:46:35.135 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:34 smithi195 ceph-mon[82388]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.135 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:34 smithi195 ceph-mon[82388]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:34 smithi195 ceph-mon[82388]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:34 smithi195 ceph-mon[82388]: pgmap v518: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 209 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2024-09-14T03:46:35.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:34 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3085412596' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:35.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:34 smithi195 ceph-mon[82388]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T03:46:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[86423]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[86423]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[86423]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[86423]: pgmap v518: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 209 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2024-09-14T03:46:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3085412596' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:35.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[86423]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T03:46:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[88262]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[88262]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[88262]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[88262]: pgmap v518: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 209 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2024-09-14T03:46:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3085412596' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:34 smithi110 ceph-mon[88262]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T03:46:35.756 INFO:tasks.rados.rados.0.smithi110.stdout:2568: done (1 left) 2024-09-14T03:46:35.757 INFO:tasks.rados.rados.0.smithi110.stdout:2569: rmattr oid 17 current snap is 263 2024-09-14T03:46:35.757 INFO:tasks.rados.rados.0.smithi110.stdout:2569: done (1 left) 2024-09-14T03:46:35.757 INFO:tasks.rados.rados.0.smithi110.stdout:2570: copy_from oid 26 from oid 25 current snap is 263 2024-09-14T03:46:35.757 INFO:tasks.rados.rados.0.smithi110.stdout:2567: expect (ObjNum 774 snap 251 seq_num 774) 2024-09-14T03:46:35.787 INFO:tasks.rados.rados.0.smithi110.stdout:2567: done (1 left) 2024-09-14T03:46:35.788 INFO:tasks.rados.rados.0.smithi110.stdout:2571: read oid 38 snap -1 2024-09-14T03:46:35.788 INFO:tasks.rados.rados.0.smithi110.stdout:2571: expect (ObjNum 718 snap 229 seq_num 718) 2024-09-14T03:46:35.788 INFO:tasks.rados.rados.0.smithi110.stdout:2572: read oid 18 snap -1 2024-09-14T03:46:35.788 INFO:tasks.rados.rados.0.smithi110.stdout:2572: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:46:35.788 INFO:tasks.rados.rados.0.smithi110.stdout:2573: read oid 46 snap 238 2024-09-14T03:46:35.788 INFO:tasks.rados.rados.0.smithi110.stdout:2573: expect (ObjNum 704 snap 225 seq_num 704) 2024-09-14T03:46:35.788 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 263 2024-09-14T03:46:35.788 INFO:tasks.rados.rados.0.smithi110.stdout:2574: seq_num 794 ranges {1204224=499712,1703936=532480,2236416=270336} 2024-09-14T03:46:35.797 INFO:tasks.rados.rados.0.smithi110.stdout:2574: writing smithi11075562-22 from 1204224 to 1703936 tid 1 2024-09-14T03:46:35.801 INFO:tasks.rados.rados.0.smithi110.stdout:2574: writing smithi11075562-22 from 1703936 to 2236416 tid 2 2024-09-14T03:46:35.803 INFO:tasks.rados.rados.0.smithi110.stdout:2574: writing smithi11075562-22 from 2236416 to 2506752 tid 3 2024-09-14T03:46:35.803 INFO:tasks.rados.rados.0.smithi110.stdout:2575: snap_create 2024-09-14T03:46:35.803 INFO:tasks.rados.rados.0.smithi110.stdout:2572: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:46:35.808 INFO:tasks.rados.rados.0.smithi110.stdout:2570: finishing copy_from racing read to smithi11075562-26 2024-09-14T03:46:35.808 INFO:tasks.rados.rados.0.smithi110.stdout:2571: expect (ObjNum 718 snap 229 seq_num 718) 2024-09-14T03:46:35.914 INFO:tasks.rados.rados.0.smithi110.stdout:2573: expect (ObjNum 704 snap 225 seq_num 704) 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:2570: finishing copy_from to smithi11075562-26 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1443 (ObjNum 642 snap 199 seq_num 642) dirty exists 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:2574: finishing write tid 1 to smithi11075562-22 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:2574: finishing write tid 2 to smithi11075562-22 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:2574: finishing write tid 3 to smithi11075562-22 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:2574: finishing write tid 4 to smithi11075562-22 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:2574: finishing write tid 5 to smithi11075562-22 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1529 (ObjNum 794 snap 263 seq_num 794) dirty exists 2024-09-14T03:46:36.007 INFO:tasks.rados.rados.0.smithi110.stdout:2574: left oid 22 (ObjNum 794 snap 263 seq_num 794) 2024-09-14T03:46:36.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:36.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.059 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:46:36.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:35 smithi195 ceph-mon[82388]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T03:46:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:46:36.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.199 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.200 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[88262]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T03:46:36.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:46:36.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:35 smithi110 ceph-mon[86423]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T03:46:36.810 INFO:tasks.rados.rados.0.smithi110.stdout:2570: done (5 left) 2024-09-14T03:46:36.811 INFO:tasks.rados.rados.0.smithi110.stdout:2571: done (4 left) 2024-09-14T03:46:36.811 INFO:tasks.rados.rados.0.smithi110.stdout:2572: done (3 left) 2024-09-14T03:46:36.811 INFO:tasks.rados.rados.0.smithi110.stdout:2573: done (2 left) 2024-09-14T03:46:36.811 INFO:tasks.rados.rados.0.smithi110.stdout:2574: done (1 left) 2024-09-14T03:46:36.811 INFO:tasks.rados.rados.0.smithi110.stdout:2575: done (0 left) 2024-09-14T03:46:36.811 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 264 2024-09-14T03:46:36.812 INFO:tasks.rados.rados.0.smithi110.stdout:2576: seq_num 795 ranges {0=524288,524288=417792,942080=385024} 2024-09-14T03:46:36.812 INFO:tasks.rados.rados.0.smithi110.stdout:2576: writing smithi11075562-9 from 0 to 524288 tid 1 2024-09-14T03:46:36.815 INFO:tasks.rados.rados.0.smithi110.stdout:2576: writing smithi11075562-9 from 524288 to 942080 tid 2 2024-09-14T03:46:36.818 INFO:tasks.rados.rados.0.smithi110.stdout:2576: writing smithi11075562-9 from 942080 to 1327104 tid 3 2024-09-14T03:46:36.818 INFO:tasks.rados.rados.0.smithi110.stdout:2577: read oid 18 snap -1 2024-09-14T03:46:36.818 INFO:tasks.rados.rados.0.smithi110.stdout:2577: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:46:36.818 INFO:tasks.rados.rados.0.smithi110.stdout:2578: snap_remove snap 246 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: Detected new or changed devices on smithi195 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all mds 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all rgw 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all nfs 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:46:37.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:36 smithi195 ceph-mon[82388]: pgmap v521: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 185 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:46:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: Detected new or changed devices on smithi195 2024-09-14T03:46:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all mds 2024-09-14T03:46:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all rgw 2024-09-14T03:46:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:46:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:46:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:46:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all nfs 2024-09-14T03:46:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[88262]: pgmap v521: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 185 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: Detected new or changed devices on smithi195 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all mds 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all rgw 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all nfs 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:46:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:36 smithi110 ceph-mon[86423]: pgmap v521: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 185 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:46:37.817 INFO:tasks.rados.rados.0.smithi110.stdout:2578: done (2 left) 2024-09-14T03:46:37.817 INFO:tasks.rados.rados.0.smithi110.stdout:2579: copy_from oid 19 from oid 12 current snap is 264 2024-09-14T03:46:37.817 INFO:tasks.rados.rados.0.smithi110.stdout:2576: finishing write tid 1 to smithi11075562-9 2024-09-14T03:46:37.817 INFO:tasks.rados.rados.0.smithi110.stdout:2577: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2576: finishing write tid 2 to smithi11075562-9 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2576: finishing write tid 3 to smithi11075562-9 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2576: finishing write tid 4 to smithi11075562-9 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2576: finishing write tid 5 to smithi11075562-9 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1582 (ObjNum 795 snap 264 seq_num 795) dirty exists 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2576: left oid 9 (ObjNum 795 snap 264 seq_num 795) 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2576: done (2 left) 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2577: done (1 left) 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2580: read oid 38 snap -1 2024-09-14T03:46:37.823 INFO:tasks.rados.rados.0.smithi110.stdout:2580: expect (ObjNum 718 snap 229 seq_num 718) 2024-09-14T03:46:37.824 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 264 2024-09-14T03:46:37.824 INFO:tasks.rados.rados.0.smithi110.stdout:2581: seq_num 796 ranges {1409024=540672,1949696=704512,2654208=98304} 2024-09-14T03:46:37.834 INFO:tasks.rados.rados.0.smithi110.stdout:2581: writing smithi11075562-4 from 1409024 to 1949696 tid 1 2024-09-14T03:46:37.838 INFO:tasks.rados.rados.0.smithi110.stdout:2581: writing smithi11075562-4 from 1949696 to 2654208 tid 2 2024-09-14T03:46:37.839 INFO:tasks.rados.rados.0.smithi110.stdout:2581: writing smithi11075562-4 from 2654208 to 2752512 tid 3 2024-09-14T03:46:37.839 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 264 2024-09-14T03:46:37.839 INFO:tasks.rados.rados.0.smithi110.stdout:2582: seq_num 797 ranges {6144000=557056,6701056=589824,7290880=212992} 2024-09-14T03:46:37.876 INFO:tasks.rados.rados.0.smithi110.stdout:2582: writing smithi11075562-1 from 6144000 to 6701056 tid 1 2024-09-14T03:46:37.879 INFO:tasks.rados.rados.0.smithi110.stdout:2582: writing smithi11075562-1 from 6701056 to 7290880 tid 2 2024-09-14T03:46:37.881 INFO:tasks.rados.rados.0.smithi110.stdout:2582: writing smithi11075562-1 from 7290880 to 7503872 tid 3 2024-09-14T03:46:37.881 INFO:tasks.rados.rados.0.smithi110.stdout:2583: delete oid 44 current snap is 264 2024-09-14T03:46:37.881 INFO:tasks.rados.rados.0.smithi110.stdout:2580: expect (ObjNum 718 snap 229 seq_num 718) 2024-09-14T03:46:37.987 INFO:tasks.rados.rados.0.smithi110.stdout:2581: finishing write tid 1 to smithi11075562-4 2024-09-14T03:46:37.987 INFO:tasks.rados.rados.0.smithi110.stdout:2580: done (4 left) 2024-09-14T03:46:37.987 INFO:tasks.rados.rados.0.smithi110.stdout:2583: done (3 left) 2024-09-14T03:46:37.987 INFO:tasks.rados.rados.0.smithi110.stdout:2584: setattr oid 31 current snap is 264 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2581: finishing write tid 2 to smithi11075562-4 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2581: finishing write tid 3 to smithi11075562-4 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2581: finishing write tid 4 to smithi11075562-4 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2581: finishing write tid 5 to smithi11075562-4 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1575 (ObjNum 796 snap 264 seq_num 796) dirty exists 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2581: left oid 4 (ObjNum 796 snap 264 seq_num 796) 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2579: finishing copy_from racing read to smithi11075562-19 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2579: finishing copy_from to smithi11075562-19 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 1497 (ObjNum 751 snap 244 seq_num 751) dirty exists 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2582: finishing write tid 1 to smithi11075562-1 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2582: finishing write tid 2 to smithi11075562-1 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2582: finishing write tid 3 to smithi11075562-1 2024-09-14T03:46:37.988 INFO:tasks.rados.rados.0.smithi110.stdout:2582: finishing write tid 4 to smithi11075562-1 2024-09-14T03:46:37.989 INFO:tasks.rados.rados.0.smithi110.stdout:2582: finishing write tid 5 to smithi11075562-1 2024-09-14T03:46:37.989 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1503 (ObjNum 797 snap 264 seq_num 797) dirty exists 2024-09-14T03:46:37.989 INFO:tasks.rados.rados.0.smithi110.stdout:2582: left oid 1 (ObjNum 797 snap 264 seq_num 797) 2024-09-14T03:46:37.989 INFO:tasks.rados.rados.0.smithi110.stdout:2579: done (3 left) 2024-09-14T03:46:37.989 INFO:tasks.rados.rados.0.smithi110.stdout:2581: done (2 left) 2024-09-14T03:46:37.989 INFO:tasks.rados.rados.0.smithi110.stdout:2582: done (1 left) 2024-09-14T03:46:37.989 INFO:tasks.rados.rados.0.smithi110.stdout:2585: read oid 1 snap -1 2024-09-14T03:46:37.990 INFO:tasks.rados.rados.0.smithi110.stdout:2585: expect (ObjNum 797 snap 264 seq_num 797) 2024-09-14T03:46:37.990 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 264 2024-09-14T03:46:37.990 INFO:tasks.rados.rados.0.smithi110.stdout:2586: seq_num 799 ranges {8306688=589824,8896512=802816} 2024-09-14T03:46:38.040 INFO:tasks.rados.rados.0.smithi110.stdout:2586: writing smithi11075562-26 from 8306688 to 8896512 tid 1 2024-09-14T03:46:38.045 INFO:tasks.rados.rados.0.smithi110.stdout:2586: writing smithi11075562-26 from 8896512 to 9699328 tid 2 2024-09-14T03:46:38.045 INFO:tasks.rados.rados.0.smithi110.stdout:2587: rollback oid 2 current snap is 264 2024-09-14T03:46:38.045 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 2 to 261 2024-09-14T03:46:38.045 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1576 (ObjNum 733 snap 233 seq_num 733) dirty exists 2024-09-14T03:46:38.045 INFO:tasks.rados.rados.0.smithi110.stdout:2585: expect (ObjNum 797 snap 264 seq_num 797) 2024-09-14T03:46:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:37 smithi110 ceph-mon[88262]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T03:46:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:37 smithi110 ceph-mon[88262]: Upgrade: Updating grafana.a 2024-09-14T03:46:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:37 smithi110 ceph-mon[88262]: Deploying daemon grafana.a on smithi195 2024-09-14T03:46:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:37 smithi110 ceph-mon[88262]: pgmap v523: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 180 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 0 B/s wr, 10 op/s 2024-09-14T03:46:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:37 smithi110 ceph-mon[86423]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T03:46:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:37 smithi110 ceph-mon[86423]: Upgrade: Updating grafana.a 2024-09-14T03:46:38.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:37 smithi110 ceph-mon[86423]: Deploying daemon grafana.a on smithi195 2024-09-14T03:46:38.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:37 smithi110 ceph-mon[86423]: pgmap v523: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 180 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 0 B/s wr, 10 op/s 2024-09-14T03:46:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:37 smithi195 ceph-mon[82388]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T03:46:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:37 smithi195 ceph-mon[82388]: Upgrade: Updating grafana.a 2024-09-14T03:46:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:37 smithi195 ceph-mon[82388]: Deploying daemon grafana.a on smithi195 2024-09-14T03:46:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:37 smithi195 ceph-mon[82388]: pgmap v523: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 180 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 0 B/s wr, 10 op/s 2024-09-14T03:46:38.479 INFO:tasks.rados.rados.0.smithi110.stdout:2586: finishing write tid 1 to smithi11075562-26 2024-09-14T03:46:38.479 INFO:tasks.rados.rados.0.smithi110.stdout:2584: done (3 left) 2024-09-14T03:46:38.479 INFO:tasks.rados.rados.0.smithi110.stdout:2585: done (2 left) 2024-09-14T03:46:38.479 INFO:tasks.rados.rados.0.smithi110.stdout:2588: setattr oid 9 current snap is 264 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:2587: finishing rollback tid 0 to smithi11075562-2 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:2587: finishing rollback tid 1 to smithi11075562-2 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:2587: finishing rollback tid 2 to smithi11075562-2 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 1646 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:2586: finishing write tid 2 to smithi11075562-26 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:2586: finishing write tid 3 to smithi11075562-26 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:2586: finishing write tid 4 to smithi11075562-26 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1446 (ObjNum 799 snap 264 seq_num 799) dirty exists 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:2586: left oid 26 (ObjNum 799 snap 264 seq_num 799) 2024-09-14T03:46:38.480 INFO:tasks.rados.rados.0.smithi110.stdout:2586: done (2 left) 2024-09-14T03:46:38.481 INFO:tasks.rados.rados.0.smithi110.stdout:2587: done (1 left) 2024-09-14T03:46:38.481 INFO:tasks.rados.rados.0.smithi110.stdout:2589: snap_create 2024-09-14T03:46:38.482 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1583 (ObjNum 795 snap 264 seq_num 795) dirty exists 2024-09-14T03:46:38.755 INFO:tasks.rados.rados.0.smithi110.stdout:2588: done (1 left) 2024-09-14T03:46:38.755 INFO:tasks.rados.rados.0.smithi110.stdout:2589: done (0 left) 2024-09-14T03:46:38.755 INFO:tasks.rados.rados.0.smithi110.stdout:2590: delete oid 27 current snap is 265 2024-09-14T03:46:38.759 INFO:tasks.rados.rados.0.smithi110.stdout:2590: done (0 left) 2024-09-14T03:46:38.759 INFO:tasks.rados.rados.0.smithi110.stdout:2591: read oid 47 snap 262 2024-09-14T03:46:38.759 INFO:tasks.rados.rados.0.smithi110.stdout:2591: expect deleted 2024-09-14T03:46:38.759 INFO:tasks.rados.rados.0.smithi110.stdout:2592: read oid 6 snap -1 2024-09-14T03:46:38.759 INFO:tasks.rados.rados.0.smithi110.stdout:2592: expect deleted 2024-09-14T03:46:38.760 INFO:tasks.rados.rados.0.smithi110.stdout:2593: snap_create 2024-09-14T03:46:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:38 smithi110 ceph-mon[88262]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T03:46:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:38 smithi110 ceph-mon[88262]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T03:46:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:38 smithi110 ceph-mon[86423]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T03:46:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:38 smithi110 ceph-mon[86423]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T03:46:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:38 smithi195 ceph-mon[82388]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T03:46:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:38 smithi195 ceph-mon[82388]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T03:46:39.757 INFO:tasks.rados.rados.0.smithi110.stdout:2591: done (2 left) 2024-09-14T03:46:39.757 INFO:tasks.rados.rados.0.smithi110.stdout:2592: done (1 left) 2024-09-14T03:46:39.757 INFO:tasks.rados.rados.0.smithi110.stdout:2593: done (0 left) 2024-09-14T03:46:39.757 INFO:tasks.rados.rados.0.smithi110.stdout:2594: copy_from oid 43 from oid 4 current snap is 266 2024-09-14T03:46:39.757 INFO:tasks.rados.rados.0.smithi110.stdout:2595: setattr oid 12 current snap is 266 2024-09-14T03:46:39.758 INFO:tasks.rados.rados.0.smithi110.stdout:2596: read oid 2 snap -1 2024-09-14T03:46:39.758 INFO:tasks.rados.rados.0.smithi110.stdout:2596: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:46:39.758 INFO:tasks.rados.rados.0.smithi110.stdout:2597: setattr oid 29 current snap is 266 2024-09-14T03:46:39.760 INFO:tasks.rados.rados.0.smithi110.stdout:2598: read oid 33 snap -1 2024-09-14T03:46:39.760 INFO:tasks.rados.rados.0.smithi110.stdout:2598: expect (ObjNum 701 snap 224 seq_num 701) 2024-09-14T03:46:39.760 INFO:tasks.rados.rados.0.smithi110.stdout:2599: setattr oid 13 current snap is 266 2024-09-14T03:46:39.762 INFO:tasks.rados.rados.0.smithi110.stdout:2600: copy_from oid 50 from oid 49 current snap is 266 2024-09-14T03:46:39.762 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 266 2024-09-14T03:46:39.763 INFO:tasks.rados.rados.0.smithi110.stdout:2601: seq_num 801 ranges {2834432=622592,3457024=573440,4030464=229376} 2024-09-14T03:46:39.783 INFO:tasks.rados.rados.0.smithi110.stdout:2601: writing smithi11075562-42 from 2834432 to 3457024 tid 1 2024-09-14T03:46:39.787 INFO:tasks.rados.rados.0.smithi110.stdout:2601: writing smithi11075562-42 from 3457024 to 4030464 tid 2 2024-09-14T03:46:39.789 INFO:tasks.rados.rados.0.smithi110.stdout:2601: writing smithi11075562-42 from 4030464 to 4259840 tid 3 2024-09-14T03:46:39.789 INFO:tasks.rados.rados.0.smithi110.stdout:2602: rollback oid 21 current snap is 266 2024-09-14T03:46:39.789 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 257 2024-09-14T03:46:39.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1783 (ObjNum 751 snap 244 seq_num 751) dirty exists 2024-09-14T03:46:39.789 INFO:tasks.rados.rados.0.smithi110.stdout:2596: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:46:39.863 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1587 (ObjNum 653 snap 202 seq_num 653) dirty exists 2024-09-14T03:46:39.863 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1505 (ObjNum 773 snap 251 seq_num 773) dirty exists 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2594: finishing copy_from racing read to smithi11075562-43 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2594: finishing copy_from to smithi11075562-43 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1575 (ObjNum 796 snap 264 seq_num 796) dirty exists 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2601: finishing write tid 1 to smithi11075562-42 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2594: done (8 left) 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2595: done (7 left) 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2596: done (6 left) 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2597: done (5 left) 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2599: done (4 left) 2024-09-14T03:46:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:2603: read oid 8 snap -1 2024-09-14T03:46:39.865 INFO:tasks.rados.rados.0.smithi110.stdout:2603: expect deleted 2024-09-14T03:46:39.865 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 37 current snap is 266 2024-09-14T03:46:39.865 INFO:tasks.rados.rados.0.smithi110.stdout:2602: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:46:39.865 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 0 (ObjNum 81 snap 0 seq_num 66615968) dirty dne 2024-09-14T03:46:39.865 INFO:tasks.rados.rados.0.smithi110.stdout:2598: expect (ObjNum 701 snap 224 seq_num 701) 2024-09-14T03:46:40.064 INFO:tasks.rados.rados.0.smithi110.stdout:2601: finishing write tid 2 to smithi11075562-42 2024-09-14T03:46:40.064 INFO:tasks.rados.rados.0.smithi110.stdout:2601: finishing write tid 3 to smithi11075562-42 2024-09-14T03:46:40.064 INFO:tasks.rados.rados.0.smithi110.stdout:2601: finishing write tid 4 to smithi11075562-42 2024-09-14T03:46:40.064 INFO:tasks.rados.rados.0.smithi110.stdout:2601: finishing write tid 5 to smithi11075562-42 2024-09-14T03:46:40.065 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1510 (ObjNum 801 snap 266 seq_num 801) dirty exists 2024-09-14T03:46:40.065 INFO:tasks.rados.rados.0.smithi110.stdout:2601: left oid 42 (ObjNum 801 snap 266 seq_num 801) 2024-09-14T03:46:40.065 INFO:tasks.rados.rados.0.smithi110.stdout:2600: finishing copy_from racing read to smithi11075562-50 2024-09-14T03:46:40.065 INFO:tasks.rados.rados.0.smithi110.stdout:2600: finishing copy_from to smithi11075562-50 2024-09-14T03:46:40.065 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 1479 (ObjNum 784 snap 257 seq_num 784) dirty exists 2024-09-14T03:46:40.065 INFO:tasks.rados.rados.0.smithi110.stdout:2604: seq_num 802 ranges {0=638976,638976=458752,1097728=344064} 2024-09-14T03:46:40.067 INFO:tasks.rados.rados.0.smithi110.stdout:2604: writing smithi11075562-37 from 0 to 638976 tid 1 2024-09-14T03:46:40.070 INFO:tasks.rados.rados.0.smithi110.stdout:2604: writing smithi11075562-37 from 638976 to 1097728 tid 2 2024-09-14T03:46:40.072 INFO:tasks.rados.rados.0.smithi110.stdout:2604: writing smithi11075562-37 from 1097728 to 1441792 tid 3 2024-09-14T03:46:40.072 INFO:tasks.rados.rados.0.smithi110.stdout:2598: done (5 left) 2024-09-14T03:46:40.072 INFO:tasks.rados.rados.0.smithi110.stdout:2600: done (4 left) 2024-09-14T03:46:40.072 INFO:tasks.rados.rados.0.smithi110.stdout:2601: done (3 left) 2024-09-14T03:46:40.072 INFO:tasks.rados.rados.0.smithi110.stdout:2602: done (2 left) 2024-09-14T03:46:40.072 INFO:tasks.rados.rados.0.smithi110.stdout:2605: read oid 26 snap -1 2024-09-14T03:46:40.072 INFO:tasks.rados.rados.0.smithi110.stdout:2605: expect (ObjNum 799 snap 264 seq_num 799) 2024-09-14T03:46:40.072 INFO:tasks.rados.rados.0.smithi110.stdout:2606: snap_create 2024-09-14T03:46:40.073 INFO:tasks.rados.rados.0.smithi110.stdout:2604: finishing write tid 1 to smithi11075562-37 2024-09-14T03:46:40.075 INFO:tasks.rados.rados.0.smithi110.stdout:2604: finishing write tid 2 to smithi11075562-37 2024-09-14T03:46:40.075 INFO:tasks.rados.rados.0.smithi110.stdout:2604: finishing write tid 3 to smithi11075562-37 2024-09-14T03:46:40.076 INFO:tasks.rados.rados.0.smithi110.stdout:2604: finishing write tid 4 to smithi11075562-37 2024-09-14T03:46:40.080 INFO:tasks.rados.rados.0.smithi110.stdout:2604: finishing write tid 5 to smithi11075562-37 2024-09-14T03:46:40.081 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1787 (ObjNum 802 snap 266 seq_num 802) dirty exists 2024-09-14T03:46:40.081 INFO:tasks.rados.rados.0.smithi110.stdout:2604: left oid 37 (ObjNum 802 snap 266 seq_num 802) 2024-09-14T03:46:40.124 INFO:tasks.rados.rados.0.smithi110.stdout:2605: expect (ObjNum 799 snap 264 seq_num 799) 2024-09-14T03:46:40.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:46:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:46:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:2603: done (3 left) 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:2604: done (2 left) 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:2605: done (1 left) 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:2606: done (0 left) 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:2607: rollback oid 45 current snap is 267 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 262 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:2608: read oid 14 snap -1 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:2608: expect (ObjNum 756 snap 244 seq_num 756) 2024-09-14T03:46:41.064 INFO:tasks.rados.rados.0.smithi110.stdout:2609: snap_remove snap 258 2024-09-14T03:46:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:40 smithi110 ceph-mon[88262]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T03:46:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:40 smithi110 ceph-mon[88262]: pgmap v527: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 178 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 324 KiB/s wr, 7 op/s 2024-09-14T03:46:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:40 smithi110 ceph-mon[86423]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T03:46:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:40 smithi110 ceph-mon[86423]: pgmap v527: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 178 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 324 KiB/s wr, 7 op/s 2024-09-14T03:46:41.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:40 smithi195 ceph-mon[82388]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T03:46:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:40 smithi195 ceph-mon[82388]: pgmap v527: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 178 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 324 KiB/s wr, 7 op/s 2024-09-14T03:46:41.767 INFO:tasks.rados.rados.0.smithi110.stdout:2609: done (2 left) 2024-09-14T03:46:41.768 INFO:tasks.rados.rados.0.smithi110.stdout:2610: read oid 15 snap 250 2024-09-14T03:46:41.768 INFO:tasks.rados.rados.0.smithi110.stdout:2610: expect deleted 2024-09-14T03:46:41.768 INFO:tasks.rados.rados.0.smithi110.stdout:2611: delete oid 13 current snap is 267 2024-09-14T03:46:41.768 INFO:tasks.rados.rados.0.smithi110.stdout:2607: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:46:41.768 INFO:tasks.rados.rados.0.smithi110.stdout:2607: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:46:41.768 INFO:tasks.rados.rados.0.smithi110.stdout:2607: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:46:41.768 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 1791 (ObjNum 649 snap 201 seq_num 649) dirty exists 2024-09-14T03:46:41.768 INFO:tasks.rados.rados.0.smithi110.stdout:2608: expect (ObjNum 756 snap 244 seq_num 756) 2024-09-14T03:46:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:41 smithi110 ceph-mon[88262]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T03:46:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:41 smithi110 ceph-mon[86423]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T03:46:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:46:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:46:41] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.51.0" 2024-09-14T03:46:42.000 INFO:tasks.rados.rados.0.smithi110.stdout:2607: done (3 left) 2024-09-14T03:46:42.000 INFO:tasks.rados.rados.0.smithi110.stdout:2608: done (2 left) 2024-09-14T03:46:42.000 INFO:tasks.rados.rados.0.smithi110.stdout:2610: done (1 left) 2024-09-14T03:46:42.000 INFO:tasks.rados.rados.0.smithi110.stdout:2611: done (0 left) 2024-09-14T03:46:42.000 INFO:tasks.rados.rados.0.smithi110.stdout:2612: rmattr oid 34 current snap is 267 2024-09-14T03:46:42.001 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 267 2024-09-14T03:46:42.001 INFO:tasks.rados.rados.0.smithi110.stdout:2613: seq_num 804 ranges {0=671744,671744=622592,1294336=180224} 2024-09-14T03:46:42.003 INFO:tasks.rados.rados.0.smithi110.stdout:2613: writing smithi11075562-6 from 0 to 671744 tid 1 2024-09-14T03:46:42.007 INFO:tasks.rados.rados.0.smithi110.stdout:2613: writing smithi11075562-6 from 671744 to 1294336 tid 2 2024-09-14T03:46:42.009 INFO:tasks.rados.rados.0.smithi110.stdout:2613: writing smithi11075562-6 from 1294336 to 1474560 tid 3 2024-09-14T03:46:42.009 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 267 2024-09-14T03:46:42.009 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1577 (ObjNum 701 snap 224 seq_num 701) dirty exists 2024-09-14T03:46:42.009 INFO:tasks.rados.rados.0.smithi110.stdout:2613: finishing write tid 1 to smithi11075562-6 2024-09-14T03:46:42.009 INFO:tasks.rados.rados.0.smithi110.stdout:2614: seq_num 805 ranges {294912=688128,983040=507904,1490944=294912} 2024-09-14T03:46:42.014 INFO:tasks.rados.rados.0.smithi110.stdout:2614: writing smithi11075562-41 from 294912 to 983040 tid 1 2024-09-14T03:46:42.017 INFO:tasks.rados.rados.0.smithi110.stdout:2614: writing smithi11075562-41 from 983040 to 1490944 tid 2 2024-09-14T03:46:42.019 INFO:tasks.rados.rados.0.smithi110.stdout:2614: writing smithi11075562-41 from 1490944 to 1785856 tid 3 2024-09-14T03:46:42.019 INFO:tasks.rados.rados.0.smithi110.stdout:2612: done (2 left) 2024-09-14T03:46:42.019 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 267 2024-09-14T03:46:42.019 INFO:tasks.rados.rados.0.smithi110.stdout:2615: seq_num 806 ranges {0=704512,704512=442368,1146880=360448} 2024-09-14T03:46:42.024 INFO:tasks.rados.rados.0.smithi110.stdout:2615: writing smithi11075562-46 from 0 to 704512 tid 1 2024-09-14T03:46:42.026 INFO:tasks.rados.rados.0.smithi110.stdout:2615: writing smithi11075562-46 from 704512 to 1146880 tid 2 2024-09-14T03:46:42.029 INFO:tasks.rados.rados.0.smithi110.stdout:2615: writing smithi11075562-46 from 1146880 to 1507328 tid 3 2024-09-14T03:46:42.029 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 267 2024-09-14T03:46:42.029 INFO:tasks.rados.rados.0.smithi110.stdout:2613: finishing write tid 2 to smithi11075562-6 2024-09-14T03:46:42.029 INFO:tasks.rados.rados.0.smithi110.stdout:2613: finishing write tid 3 to smithi11075562-6 2024-09-14T03:46:42.029 INFO:tasks.rados.rados.0.smithi110.stdout:2616: seq_num 807 ranges {4317184=720896,5038080=729088,5767168=73728} 2024-09-14T03:46:42.056 INFO:tasks.rados.rados.0.smithi110.stdout:2616: writing smithi11075562-49 from 4317184 to 5038080 tid 1 2024-09-14T03:46:42.060 INFO:tasks.rados.rados.0.smithi110.stdout:2616: writing smithi11075562-49 from 5038080 to 5767168 tid 2 2024-09-14T03:46:42.061 INFO:tasks.rados.rados.0.smithi110.stdout:2616: writing smithi11075562-49 from 5767168 to 5840896 tid 3 2024-09-14T03:46:42.061 INFO:tasks.rados.rados.0.smithi110.stdout:2617: read oid 16 snap 238 2024-09-14T03:46:42.061 INFO:tasks.rados.rados.0.smithi110.stdout:2617: expect deleted 2024-09-14T03:46:42.061 INFO:tasks.rados.rados.0.smithi110.stdout:2618: snap_remove snap 265 2024-09-14T03:46:42.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:41 smithi195 ceph-mon[82388]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T03:46:42.772 INFO:tasks.rados.rados.0.smithi110.stdout:2618: done (5 left) 2024-09-14T03:46:42.772 INFO:tasks.rados.rados.0.smithi110.stdout:2619: snap_create 2024-09-14T03:46:42.772 INFO:tasks.rados.rados.0.smithi110.stdout:2613: finishing write tid 4 to smithi11075562-6 2024-09-14T03:46:42.772 INFO:tasks.rados.rados.0.smithi110.stdout:2613: finishing write tid 5 to smithi11075562-6 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1501 (ObjNum 804 snap 267 seq_num 804) dirty exists 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2613: left oid 6 (ObjNum 804 snap 267 seq_num 804) 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2614: finishing write tid 1 to smithi11075562-41 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2614: finishing write tid 2 to smithi11075562-41 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2614: finishing write tid 3 to smithi11075562-41 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2614: finishing write tid 4 to smithi11075562-41 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2614: finishing write tid 5 to smithi11075562-41 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1579 (ObjNum 805 snap 267 seq_num 805) dirty exists 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2614: left oid 41 (ObjNum 805 snap 267 seq_num 805) 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2615: finishing write tid 1 to smithi11075562-46 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2615: finishing write tid 2 to smithi11075562-46 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2615: finishing write tid 3 to smithi11075562-46 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2615: finishing write tid 4 to smithi11075562-46 2024-09-14T03:46:42.773 INFO:tasks.rados.rados.0.smithi110.stdout:2615: finishing write tid 5 to smithi11075562-46 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1587 (ObjNum 806 snap 267 seq_num 806) dirty exists 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:2615: left oid 46 (ObjNum 806 snap 267 seq_num 806) 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:2616: finishing write tid 1 to smithi11075562-49 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:2616: finishing write tid 2 to smithi11075562-49 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:2616: finishing write tid 3 to smithi11075562-49 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:2616: finishing write tid 4 to smithi11075562-49 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:2616: finishing write tid 5 to smithi11075562-49 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1523 (ObjNum 807 snap 267 seq_num 807) dirty exists 2024-09-14T03:46:42.774 INFO:tasks.rados.rados.0.smithi110.stdout:2616: left oid 49 (ObjNum 807 snap 267 seq_num 807) 2024-09-14T03:46:43.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:42 smithi195 ceph-mon[82388]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T03:46:43.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:42 smithi195 ceph-mon[82388]: pgmap v530: 121 pgs: 121 active+clean; 199 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2024-09-14T03:46:43.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:42 smithi110 ceph-mon[86423]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T03:46:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:42 smithi110 ceph-mon[86423]: pgmap v530: 121 pgs: 121 active+clean; 199 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2024-09-14T03:46:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:42 smithi110 ceph-mon[88262]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T03:46:43.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:42 smithi110 ceph-mon[88262]: pgmap v530: 121 pgs: 121 active+clean; 199 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2024-09-14T03:46:43.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:43.757 INFO:tasks.rados.rados.0.smithi110.stdout:2613: done (5 left) 2024-09-14T03:46:43.757 INFO:tasks.rados.rados.0.smithi110.stdout:2614: done (4 left) 2024-09-14T03:46:43.758 INFO:tasks.rados.rados.0.smithi110.stdout:2615: done (3 left) 2024-09-14T03:46:43.758 INFO:tasks.rados.rados.0.smithi110.stdout:2616: done (2 left) 2024-09-14T03:46:43.758 INFO:tasks.rados.rados.0.smithi110.stdout:2617: done (1 left) 2024-09-14T03:46:43.758 INFO:tasks.rados.rados.0.smithi110.stdout:2619: done (0 left) 2024-09-14T03:46:43.758 INFO:tasks.rados.rados.0.smithi110.stdout:2620: snap_create 2024-09-14T03:46:44.124 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]} : dispatch 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"} : dispatch 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]} : dispatch 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"} : dispatch 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"} : dispatch 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"} : dispatch 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]': finished 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"}]': finished 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"}]': finished 2024-09-14T03:46:44.125 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:43 smithi195 ceph-mon[82388]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T03:46:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T03:46:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]} : dispatch 2024-09-14T03:46:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"} : dispatch 2024-09-14T03:46:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]} : dispatch 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"} : dispatch 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"} : dispatch 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"} : dispatch 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]': finished 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"}]': finished 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"}]': finished 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[86423]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]} : dispatch 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"} : dispatch 2024-09-14T03:46:44.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]} : dispatch 2024-09-14T03:46:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"} : dispatch 2024-09-14T03:46:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"} : dispatch 2024-09-14T03:46:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"} : dispatch 2024-09-14T03:46:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]': finished 2024-09-14T03:46:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.1"}]': finished 2024-09-14T03:46:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "2.13"}]': finished 2024-09-14T03:46:44.198 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:43 smithi110 ceph-mon[88262]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T03:46:44.759 INFO:tasks.rados.rados.0.smithi110.stdout:2620: done (0 left) 2024-09-14T03:46:44.759 INFO:tasks.rados.rados.0.smithi110.stdout:2621: delete oid 40 current snap is 269 2024-09-14T03:46:44.764 INFO:tasks.rados.rados.0.smithi110.stdout:2621: done (0 left) 2024-09-14T03:46:44.764 INFO:tasks.rados.rados.0.smithi110.stdout:2622: snap_remove snap 256 2024-09-14T03:46:45.087 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:44 smithi195 ceph-mon[82388]: pgmap v533: 121 pgs: 121 active+clean; 204 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 6.6 MiB/s wr, 42 op/s 2024-09-14T03:46:45.087 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:44 smithi195 ceph-mon[82388]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T03:46:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:44 smithi110 ceph-mon[88262]: pgmap v533: 121 pgs: 121 active+clean; 204 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 6.6 MiB/s wr, 42 op/s 2024-09-14T03:46:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:44 smithi110 ceph-mon[88262]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T03:46:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:44 smithi110 ceph-mon[86423]: pgmap v533: 121 pgs: 121 active+clean; 204 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 6.6 MiB/s wr, 42 op/s 2024-09-14T03:46:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:44 smithi110 ceph-mon[86423]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T03:46:45.762 INFO:tasks.rados.rados.0.smithi110.stdout:2622: done (0 left) 2024-09-14T03:46:45.762 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 269 2024-09-14T03:46:45.762 INFO:tasks.rados.rados.0.smithi110.stdout:2623: seq_num 809 ranges {5988352=753664,6742016=499712,7241728=303104} 2024-09-14T03:46:45.799 INFO:tasks.rados.rados.0.smithi110.stdout:2623: writing smithi11075562-5 from 5988352 to 6742016 tid 1 2024-09-14T03:46:45.801 INFO:tasks.rados.rados.0.smithi110.stdout:2623: writing smithi11075562-5 from 6742016 to 7241728 tid 2 2024-09-14T03:46:45.803 INFO:tasks.rados.rados.0.smithi110.stdout:2623: writing smithi11075562-5 from 7241728 to 7544832 tid 3 2024-09-14T03:46:45.803 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 269 2024-09-14T03:46:45.803 INFO:tasks.rados.rados.0.smithi110.stdout:2624: seq_num 810 ranges {0=770048,770048=778240,1548288=24576} 2024-09-14T03:46:45.807 INFO:tasks.rados.rados.0.smithi110.stdout:2624: writing smithi11075562-24 from 0 to 770048 tid 1 2024-09-14T03:46:45.813 INFO:tasks.rados.rados.0.smithi110.stdout:2624: writing smithi11075562-24 from 770048 to 1548288 tid 2 2024-09-14T03:46:45.813 INFO:tasks.rados.rados.0.smithi110.stdout:2624: writing smithi11075562-24 from 1548288 to 1572864 tid 3 2024-09-14T03:46:45.813 INFO:tasks.rados.rados.0.smithi110.stdout:2625: rollback oid 13 current snap is 269 2024-09-14T03:46:45.813 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 268 2024-09-14T03:46:45.813 INFO:tasks.rados.rados.0.smithi110.stdout:2623: finishing write tid 1 to smithi11075562-5 2024-09-14T03:46:45.813 INFO:tasks.rados.rados.0.smithi110.stdout:2623: finishing write tid 2 to smithi11075562-5 2024-09-14T03:46:45.813 INFO:tasks.rados.rados.0.smithi110.stdout:2623: finishing write tid 3 to smithi11075562-5 2024-09-14T03:46:45.813 INFO:tasks.rados.rados.0.smithi110.stdout:2623: finishing write tid 4 to smithi11075562-5 2024-09-14T03:46:45.814 INFO:tasks.rados.rados.0.smithi110.stdout:2623: finishing write tid 5 to smithi11075562-5 2024-09-14T03:46:45.814 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1808 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:46:45.814 INFO:tasks.rados.rados.0.smithi110.stdout:2623: left oid 5 (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:46:45.814 INFO:tasks.rados.rados.0.smithi110.stdout:2624: finishing write tid 1 to smithi11075562-24 2024-09-14T03:46:45.814 INFO:tasks.rados.rados.0.smithi110.stdout:2623: done (2 left) 2024-09-14T03:46:45.814 INFO:tasks.rados.rados.0.smithi110.stdout:2626: delete oid 45 current snap is 269 2024-09-14T03:46:45.815 INFO:tasks.rados.rados.0.smithi110.stdout:2626: done (2 left) 2024-09-14T03:46:45.815 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 269 2024-09-14T03:46:45.815 INFO:tasks.rados.rados.0.smithi110.stdout:2627: seq_num 812 ranges {1327104=802816,2129920=548864,2678784=253952} 2024-09-14T03:46:45.827 INFO:tasks.rados.rados.0.smithi110.stdout:2627: writing smithi11075562-9 from 1327104 to 2129920 tid 1 2024-09-14T03:46:45.829 INFO:tasks.rados.rados.0.smithi110.stdout:2627: writing smithi11075562-9 from 2129920 to 2678784 tid 2 2024-09-14T03:46:45.831 INFO:tasks.rados.rados.0.smithi110.stdout:2627: writing smithi11075562-9 from 2678784 to 2932736 tid 3 2024-09-14T03:46:45.831 INFO:tasks.rados.rados.0.smithi110.stdout:2628: delete oid 44 current snap is 269 2024-09-14T03:46:45.831 INFO:tasks.rados.rados.0.smithi110.stdout:2625: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:46:45.832 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1516 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:46:45.832 INFO:tasks.rados.rados.0.smithi110.stdout:2624: finishing write tid 2 to smithi11075562-24 2024-09-14T03:46:45.832 INFO:tasks.rados.rados.0.smithi110.stdout:2624: finishing write tid 3 to smithi11075562-24 2024-09-14T03:46:45.832 INFO:tasks.rados.rados.0.smithi110.stdout:2624: finishing write tid 4 to smithi11075562-24 2024-09-14T03:46:45.832 INFO:tasks.rados.rados.0.smithi110.stdout:2624: finishing write tid 5 to smithi11075562-24 2024-09-14T03:46:45.832 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1500 (ObjNum 810 snap 269 seq_num 810) dirty exists 2024-09-14T03:46:45.832 INFO:tasks.rados.rados.0.smithi110.stdout:2624: left oid 24 (ObjNum 810 snap 269 seq_num 810) 2024-09-14T03:46:45.833 INFO:tasks.rados.rados.0.smithi110.stdout:2627: finishing write tid 1 to smithi11075562-9 2024-09-14T03:46:45.833 INFO:tasks.rados.rados.0.smithi110.stdout:2624: done (3 left) 2024-09-14T03:46:45.833 INFO:tasks.rados.rados.0.smithi110.stdout:2625: done (2 left) 2024-09-14T03:46:45.833 INFO:tasks.rados.rados.0.smithi110.stdout:2628: done (1 left) 2024-09-14T03:46:45.833 INFO:tasks.rados.rados.0.smithi110.stdout:2629: snap_remove snap 239 2024-09-14T03:46:46.767 INFO:tasks.rados.rados.0.smithi110.stdout:2629: done (1 left) 2024-09-14T03:46:46.768 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 269 2024-09-14T03:46:46.768 INFO:tasks.rados.rados.0.smithi110.stdout:2630: seq_num 814 ranges {2236416=442368,2678784=770048,3448832=434176} 2024-09-14T03:46:46.783 INFO:tasks.rados.rados.0.smithi110.stdout:2630: writing smithi11075562-38 from 2236416 to 2678784 tid 1 2024-09-14T03:46:46.788 INFO:tasks.rados.rados.0.smithi110.stdout:2630: writing smithi11075562-38 from 2678784 to 3448832 tid 2 2024-09-14T03:46:46.791 INFO:tasks.rados.rados.0.smithi110.stdout:2630: writing smithi11075562-38 from 3448832 to 3883008 tid 3 2024-09-14T03:46:46.791 INFO:tasks.rados.rados.0.smithi110.stdout:2631: snap_remove snap 260 2024-09-14T03:46:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:46 smithi110 ceph-mon[86423]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T03:46:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:46 smithi110 ceph-mon[86423]: pgmap v536: 121 pgs: 2 active+clean+snaptrim_wait, 1 peering, 118 active+clean; 208 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 20 op/s 2024-09-14T03:46:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:46 smithi110 ceph-mon[88262]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T03:46:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:46 smithi110 ceph-mon[88262]: pgmap v536: 121 pgs: 2 active+clean+snaptrim_wait, 1 peering, 118 active+clean; 208 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 20 op/s 2024-09-14T03:46:47.198 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:46 smithi195 ceph-mon[82388]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T03:46:47.199 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:46 smithi195 ceph-mon[82388]: pgmap v536: 121 pgs: 2 active+clean+snaptrim_wait, 1 peering, 118 active+clean; 208 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 20 op/s 2024-09-14T03:46:47.772 INFO:tasks.rados.rados.0.smithi110.stdout:2627: finishing write tid 2 to smithi11075562-9 2024-09-14T03:46:47.772 INFO:tasks.rados.rados.0.smithi110.stdout:2627: finishing write tid 3 to smithi11075562-9 2024-09-14T03:46:47.772 INFO:tasks.rados.rados.0.smithi110.stdout:2627: finishing write tid 4 to smithi11075562-9 2024-09-14T03:46:47.772 INFO:tasks.rados.rados.0.smithi110.stdout:2627: finishing write tid 5 to smithi11075562-9 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1591 (ObjNum 812 snap 269 seq_num 812) dirty exists 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2627: left oid 9 (ObjNum 812 snap 269 seq_num 812) 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2630: finishing write tid 1 to smithi11075562-38 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2630: finishing write tid 2 to smithi11075562-38 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2630: finishing write tid 3 to smithi11075562-38 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2630: finishing write tid 4 to smithi11075562-38 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2630: finishing write tid 5 to smithi11075562-38 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1591 (ObjNum 814 snap 269 seq_num 814) dirty exists 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2630: left oid 38 (ObjNum 814 snap 269 seq_num 814) 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2627: done (2 left) 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2630: done (1 left) 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2631: done (0 left) 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2632: copy_from oid 16 from oid 18 current snap is 269 2024-09-14T03:46:47.773 INFO:tasks.rados.rados.0.smithi110.stdout:2633: read oid 42 snap 262 2024-09-14T03:46:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:2633: expect (ObjNum 741 snap 238 seq_num 741) 2024-09-14T03:46:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:2634: copy_from oid 10 from oid 28 current snap is 269 2024-09-14T03:46:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:2635: rmattr oid 37 current snap is 269 2024-09-14T03:46:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:2635: done (3 left) 2024-09-14T03:46:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:2636: rmattr oid 22 current snap is 269 2024-09-14T03:46:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:2636: done (3 left) 2024-09-14T03:46:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:2637: snap_create 2024-09-14T03:46:47.774 INFO:tasks.rados.rados.0.smithi110.stdout:2632: finishing copy_from racing read to smithi11075562-16 2024-09-14T03:46:47.777 INFO:tasks.rados.rados.0.smithi110.stdout:2632: finishing copy_from to smithi11075562-16 2024-09-14T03:46:47.777 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 1448 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:46:47.787 INFO:tasks.rados.rados.0.smithi110.stdout:2634: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:46:47.789 INFO:tasks.rados.rados.0.smithi110.stdout:2634: finishing copy_from to smithi11075562-10 2024-09-14T03:46:47.790 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 817 (ObjNum 774 snap 251 seq_num 774) dirty exists 2024-09-14T03:46:47.798 INFO:tasks.rados.rados.0.smithi110.stdout:2633: expect (ObjNum 741 snap 238 seq_num 741) 2024-09-14T03:46:47.832 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:47 smithi195 systemd[1]: Stopping Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:46:47.832 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:47 smithi195 ceph-mon[82388]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:46:47.832 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:47 smithi195 ceph-mon[82388]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T03:46:48.113 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:47 smithi195 podman[108702]: 2024-09-14 03:46:47.829938529 +0000 UTC m=+0.133515083 container died 03236d3bc2106cf24f112656d1d2dea19b5a2b65e8d8e21bf21375c492a44b93 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, vcs-type=git, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.tags=base rhel8, version=8.5, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-14T03:46:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:47 smithi110 ceph-mon[86423]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:46:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:47 smithi110 ceph-mon[86423]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T03:46:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:47 smithi110 ceph-mon[88262]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:46:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:47 smithi110 ceph-mon[88262]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T03:46:48.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:48 smithi195 podman[108702]: 2024-09-14 03:46:48.211165523 +0000 UTC m=+0.514742087 container cleanup 03236d3bc2106cf24f112656d1d2dea19b5a2b65e8d8e21bf21375c492a44b93 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, architecture=x86_64, io.openshift.expose-services=, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, vcs-type=git, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, name=ubi8, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public) 2024-09-14T03:46:48.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:48 smithi195 bash[108702]: ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a 2024-09-14T03:46:48.758 INFO:tasks.rados.rados.0.smithi110.stdout:2632: done (3 left) 2024-09-14T03:46:48.758 INFO:tasks.rados.rados.0.smithi110.stdout:2633: done (2 left) 2024-09-14T03:46:48.758 INFO:tasks.rados.rados.0.smithi110.stdout:2634: done (1 left) 2024-09-14T03:46:48.758 INFO:tasks.rados.rados.0.smithi110.stdout:2637: done (0 left) 2024-09-14T03:46:48.758 INFO:tasks.rados.rados.0.smithi110.stdout:2638: read oid 19 snap -1 2024-09-14T03:46:48.759 INFO:tasks.rados.rados.0.smithi110.stdout:2638: expect (ObjNum 751 snap 244 seq_num 751) 2024-09-14T03:46:48.759 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 270 2024-09-14T03:46:48.759 INFO:tasks.rados.rados.0.smithi110.stdout:2639: seq_num 815 ranges {1785856=458752,2244608=655360,2899968=409600,3309568=139264} 2024-09-14T03:46:48.770 INFO:tasks.rados.rados.0.smithi110.stdout:2639: writing smithi11075562-41 from 1785856 to 2244608 tid 1 2024-09-14T03:46:48.775 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:48 smithi195 podman[108714]: 2024-09-14 03:46:48.479137283 +0000 UTC m=+0.647389205 container remove 03236d3bc2106cf24f112656d1d2dea19b5a2b65e8d8e21bf21375c492a44b93 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, release=239.1651231664, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, name=ubi8) 2024-09-14T03:46:48.775 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:48 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:46:48.775 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:48 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-14T03:46:48.775 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:48 smithi195 systemd[1]: Stopped Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:46:48.775 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:48 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service: Consumed 6.909s CPU time. 2024-09-14T03:46:48.775 INFO:tasks.rados.rados.0.smithi110.stdout:2639: writing smithi11075562-41 from 2244608 to 2899968 tid 2 2024-09-14T03:46:48.778 INFO:tasks.rados.rados.0.smithi110.stdout:2639: writing smithi11075562-41 from 2899968 to 3309568 tid 3 2024-09-14T03:46:48.779 INFO:tasks.rados.rados.0.smithi110.stdout:2639: writing smithi11075562-41 from 3309568 to 3448832 tid 4 2024-09-14T03:46:48.779 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 270 2024-09-14T03:46:48.779 INFO:tasks.rados.rados.0.smithi110.stdout:2640: seq_num 816 ranges {368640=475136,843776=540672,1384448=614400,1998848=49152} 2024-09-14T03:46:48.785 INFO:tasks.rados.rados.0.smithi110.stdout:2640: writing smithi11075562-7 from 368640 to 843776 tid 1 2024-09-14T03:46:48.788 INFO:tasks.rados.rados.0.smithi110.stdout:2640: writing smithi11075562-7 from 843776 to 1384448 tid 2 2024-09-14T03:46:48.792 INFO:tasks.rados.rados.0.smithi110.stdout:2640: writing smithi11075562-7 from 1384448 to 1998848 tid 3 2024-09-14T03:46:48.793 INFO:tasks.rados.rados.0.smithi110.stdout:2640: writing smithi11075562-7 from 1998848 to 2048000 tid 4 2024-09-14T03:46:48.793 INFO:tasks.rados.rados.0.smithi110.stdout:2641: snap_create 2024-09-14T03:46:48.793 INFO:tasks.rados.rados.0.smithi110.stdout:2639: finishing write tid 1 to smithi11075562-41 2024-09-14T03:46:48.794 INFO:tasks.rados.rados.0.smithi110.stdout:2639: finishing write tid 2 to smithi11075562-41 2024-09-14T03:46:48.794 INFO:tasks.rados.rados.0.smithi110.stdout:2639: finishing write tid 3 to smithi11075562-41 2024-09-14T03:46:48.794 INFO:tasks.rados.rados.0.smithi110.stdout:2639: finishing write tid 4 to smithi11075562-41 2024-09-14T03:46:48.794 INFO:tasks.rados.rados.0.smithi110.stdout:2639: finishing write tid 5 to smithi11075562-41 2024-09-14T03:46:48.794 INFO:tasks.rados.rados.0.smithi110.stdout:2639: finishing write tid 6 to smithi11075562-41 2024-09-14T03:46:48.794 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1584 (ObjNum 815 snap 270 seq_num 815) dirty exists 2024-09-14T03:46:48.794 INFO:tasks.rados.rados.0.smithi110.stdout:2639: left oid 41 (ObjNum 815 snap 270 seq_num 815) 2024-09-14T03:46:48.794 INFO:tasks.rados.rados.0.smithi110.stdout:2638: expect (ObjNum 751 snap 244 seq_num 751) 2024-09-14T03:46:49.062 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:48 smithi195 systemd[1]: Starting Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:46:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:48 smithi195 ceph-mon[82388]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T03:46:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:48 smithi195 ceph-mon[82388]: pgmap v539: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 1 peering, 116 active+clean; 213 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-14T03:46:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:48 smithi195 ceph-mon[82388]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T03:46:49.107 INFO:tasks.rados.rados.0.smithi110.stdout:2640: finishing write tid 1 to smithi11075562-7 2024-09-14T03:46:49.107 INFO:tasks.rados.rados.0.smithi110.stdout:2640: finishing write tid 2 to smithi11075562-7 2024-09-14T03:46:49.107 INFO:tasks.rados.rados.0.smithi110.stdout:2640: finishing write tid 3 to smithi11075562-7 2024-09-14T03:46:49.107 INFO:tasks.rados.rados.0.smithi110.stdout:2640: finishing write tid 4 to smithi11075562-7 2024-09-14T03:46:49.108 INFO:tasks.rados.rados.0.smithi110.stdout:2640: finishing write tid 5 to smithi11075562-7 2024-09-14T03:46:49.108 INFO:tasks.rados.rados.0.smithi110.stdout:2640: finishing write tid 6 to smithi11075562-7 2024-09-14T03:46:49.108 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1596 (ObjNum 816 snap 270 seq_num 816) dirty exists 2024-09-14T03:46:49.108 INFO:tasks.rados.rados.0.smithi110.stdout:2640: left oid 7 (ObjNum 816 snap 270 seq_num 816) 2024-09-14T03:46:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:48 smithi110 ceph-mon[88262]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T03:46:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:48 smithi110 ceph-mon[88262]: pgmap v539: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 1 peering, 116 active+clean; 213 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-14T03:46:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:48 smithi110 ceph-mon[88262]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T03:46:49.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:48 smithi110 ceph-mon[86423]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T03:46:49.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:48 smithi110 ceph-mon[86423]: pgmap v539: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 1 peering, 116 active+clean; 213 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-14T03:46:49.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:48 smithi110 ceph-mon[86423]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T03:46:49.350 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 podman[108844]: 2024-09-14 03:46:48.9643506 +0000 UTC m=+0.016645275 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-09-14T03:46:49.351 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 podman[108844]: 2024-09-14 03:46:49.130232213 +0000 UTC m=+0.182526926 container create c9fbdb64fe960ef2581c07b6668b171dac3f11436e566d85b3b5f5a999e16b6a (image=quay.io/ceph/grafana:10.4.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:46:49.605 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 podman[108844]: 2024-09-14 03:46:49.350155632 +0000 UTC m=+0.402450305 container init c9fbdb64fe960ef2581c07b6668b171dac3f11436e566d85b3b5f5a999e16b6a (image=quay.io/ceph/grafana:10.4.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:46:49.605 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 podman[108844]: 2024-09-14 03:46:49.353614619 +0000 UTC m=+0.405909296 container start c9fbdb64fe960ef2581c07b6668b171dac3f11436e566d85b3b5f5a999e16b6a (image=quay.io/ceph/grafana:10.4.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473134841Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-09-14T03:46:49Z 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473351051Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473358918Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473364628Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473369457Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.47337221Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473374939Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473377567Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473380605Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473383459Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:46:49.606 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473385925Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473388455Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473391151Z level=info msg=Target target=[all] 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473397105Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473399637Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473402171Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473404684Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473407419Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=settings t=2024-09-14T03:46:49.473409924Z level=info msg="App mode production" 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=sqlstore t=2024-09-14T03:46:49.473650812Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=sqlstore t=2024-09-14T03:46:49.473663561Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.474133164Z level=info msg="Starting DB migrations" 2024-09-14T03:46:49.607 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.485438091Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.490861483Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=5.422398ms 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.491271973Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.491394074Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=122.248µs 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.491747177Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.492457524Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=710.059µs 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.492964334Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.498031711Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.066696ms 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.498499313Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.503689511Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.190055ms 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.504082019Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-14T03:46:49.608 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.509093959Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.011341ms 2024-09-14T03:46:49.609 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.509403328Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.509515893Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=112.593µs 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.509863264Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.510570077Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=705.609µs 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.510927937Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.511596581Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=668.103µs 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.511949157Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.517229116Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.279273ms 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.517588638Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.522551966Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=4.961671ms 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.522900387Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.522938939Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=37.793µs 2024-09-14T03:46:49.610 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.523326429Z level=info msg="Executing migration" id="permission kind migration" 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.528323993Z level=info msg="Migration successfully executed" id="permission kind migration" duration=4.996718ms 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.532414695Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.537449159Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.034736ms 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.537782149Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.54316126Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.378754ms 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.54348011Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.544266452Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=786.156µs 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.544586299Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.545351096Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=764.398µs 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.545676364Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-14T03:46:49.611 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.546373943Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=697.797µs 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.546691604Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.546710764Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=19.447µs 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.547095122Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.547810948Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=715.217µs 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.548109531Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.548837212Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=727.951µs 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.549135664Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.56428656Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=15.150654ms 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.564648046Z level=info msg="Executing migration" id="create correlation v2" 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.565328091Z level=info msg="Migration successfully executed" id="create correlation v2" duration=679.792µs 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.565660322Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-14T03:46:49.612 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.566366493Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=707.206µs 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.566720123Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.567472102Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=752.093µs 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.567822356Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.568512954Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=690.038µs 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.568854112Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.569023811Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=169.663µs 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.569353173Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.569929457Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=575.953µs 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.570230834Z level=info msg="Executing migration" id="add provisioning column" 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.57541775Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.186651ms 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.575755599Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-14T03:46:49.613 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.575890306Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=134.396µs 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.576226191Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.576264628Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=38.242µs 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.576598724Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.582031287Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.431882ms 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.582338012Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.582522677Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=184.75µs 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.582892403Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.583001112Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=107.508µs 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.583328675Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.584228432Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=900.534µs 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.584548544Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.585240551Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=691.449µs 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.58554969Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-14T03:46:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.586302982Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=753.097µs 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.586635136Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.587019683Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=385.38µs 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.587333288Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.587516607Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=183.272µs 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.587859948Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.588520291Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=660.062µs 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.58883382Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.589549189Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=714.584µs 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.589890534Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.590578032Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=687.373µs 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.590949917Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-14T03:46:49.615 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.591669481Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=719.282µs 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.591972669Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.592659142Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=685.841µs 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.592964925Z level=info msg="Executing migration" id="create anon_device table" 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.59351206Z level=info msg="Migration successfully executed" id="create anon_device table" duration=546.812µs 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.593839203Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.594507109Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=667.816µs 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.594808692Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.595459991Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=651.144µs 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.595780275Z level=info msg="Executing migration" id="create signing_key table" 2024-09-14T03:46:49.616 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.596328364Z level=info msg="Migration successfully executed" id="create signing_key table" duration=547.972µs 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.596634554Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.597294277Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=659.606µs 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.597585806Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.598261183Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=674.898µs 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.598575175Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.59900556Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=430.983µs 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.599360849Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.604475742Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.114694ms 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.604858295Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.606753056Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.896084ms 2024-09-14T03:46:49.617 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.607280088Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-14T03:46:49.779 INFO:tasks.rados.rados.0.smithi110.stdout:2638: done (3 left) 2024-09-14T03:46:49.779 INFO:tasks.rados.rados.0.smithi110.stdout:2639: done (2 left) 2024-09-14T03:46:49.779 INFO:tasks.rados.rados.0.smithi110.stdout:2640: done (1 left) 2024-09-14T03:46:49.779 INFO:tasks.rados.rados.0.smithi110.stdout:2641: done (0 left) 2024-09-14T03:46:49.779 INFO:tasks.rados.rados.0.smithi110.stdout:2642: read oid 29 snap -1 2024-09-14T03:46:49.779 INFO:tasks.rados.rados.0.smithi110.stdout:2642: expect (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:46:49.779 INFO:tasks.rados.rados.0.smithi110.stdout:2643: delete oid 2 current snap is 271 2024-09-14T03:46:49.785 INFO:tasks.rados.rados.0.smithi110.stdout:2643: done (1 left) 2024-09-14T03:46:49.785 INFO:tasks.rados.rados.0.smithi110.stdout:2644: setattr oid 49 current snap is 271 2024-09-14T03:46:49.786 INFO:tasks.rados.rados.0.smithi110.stdout:2642: expect (ObjNum 653 snap 202 seq_num 653) 2024-09-14T03:46:49.832 INFO:tasks.rados.rados.0.smithi110.stdout:2642: done (1 left) 2024-09-14T03:46:49.832 INFO:tasks.rados.rados.0.smithi110.stdout:2645: snap_remove snap 269 2024-09-14T03:46:49.872 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.60815425Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=873.798µs 2024-09-14T03:46:49.872 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.608459344Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-14T03:46:49.872 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.609157832Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=698.366µs 2024-09-14T03:46:49.872 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.609449201Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.610116444Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=667.024µs 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.610408771Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.61122491Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=816.039µs 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.611521748Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.612294935Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=772.829µs 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.612592429Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.61331216Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=719.519µs 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.613662862Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.614257281Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=594.328µs 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.614542794Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-14T03:46:49.873 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.614926643Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=385.702µs 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.615210473Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.6152474Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=37.385µs 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.615624885Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.620892384Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.26713ms 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.621219864Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.626518363Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.298331ms 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.626830425Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.627032731Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=202.213µs 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=migrator t=2024-09-14T03:46:49.627344722Z level=info msg="migrations completed" performed=65 skipped=482 duration=141.957994ms 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=sqlstore t=2024-09-14T03:46:49.628009303Z level=info msg="Created default organization" 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=secrets t=2024-09-14T03:46:49.629996319Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=plugin.store t=2024-09-14T03:46:49.644446383Z level=info msg="Loading plugins..." 2024-09-14T03:46:49.874 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=local.finder t=2024-09-14T03:46:49.68725372Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-14T03:46:49.875 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=plugin.store t=2024-09-14T03:46:49.687272487Z level=info msg="Plugins loaded" count=55 duration=42.826753ms 2024-09-14T03:46:49.875 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=query_data t=2024-09-14T03:46:49.689276411Z level=info msg="Query Service initialization" 2024-09-14T03:46:49.875 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=live.push_http t=2024-09-14T03:46:49.69185055Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:46:49.875 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=ngalert.migration t=2024-09-14T03:46:49.692838458Z level=info msg=Starting 2024-09-14T03:46:49.875 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=ngalert.state.manager t=2024-09-14T03:46:49.698044533Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-14T03:46:49.875 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=infra.usagestats.collector t=2024-09-14T03:46:49.699323427Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:46:49.875 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=provisioning.datasources t=2024-09-14T03:46:49.701591616Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-14T03:46:49.875 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=provisioning.datasources t=2024-09-14T03:46:49.701851241Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=provisioning.alerting t=2024-09-14T03:46:49.726499307Z level=info msg="starting to provision alerting" 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=provisioning.alerting t=2024-09-14T03:46:49.726511862Z level=info msg="finished to provision alerting" 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=grafanaStorageLogger t=2024-09-14T03:46:49.726665893Z level=info msg="Storage starting" 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=http.server t=2024-09-14T03:46:49.7285904Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=http.server t=2024-09-14T03:46:49.72884591Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=ngalert.state.manager t=2024-09-14T03:46:49.728895753Z level=info msg="Warming state cache for startup" 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:46:49.737474199Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=provisioning.dashboard t=2024-09-14T03:46:49.745484849Z level=info msg="starting to provision dashboards" 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 bash[108844]: c9fbdb64fe960ef2581c07b6668b171dac3f11436e566d85b3b5f5a999e16b6a 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 systemd[1]: Started Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=ngalert.state.manager t=2024-09-14T03:46:49.771721476Z level=info msg="State cache has been initialized" states=0 duration=42.822304ms 2024-09-14T03:46:49.876 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=ngalert.scheduler t=2024-09-14T03:46:49.771969343Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-14T03:46:49.877 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=ticker t=2024-09-14T03:46:49.772138357Z level=info msg=starting first_tick=2024-09-14T03:46:50Z 2024-09-14T03:46:49.877 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=grafana.update.checker t=2024-09-14T03:46:49.813640085Z level=info msg="Update check succeeded" duration=87.017378ms 2024-09-14T03:46:49.877 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=plugins.update.checker t=2024-09-14T03:46:49.816369741Z level=info msg="Update check succeeded" duration=89.73868ms 2024-09-14T03:46:49.877 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=sqlstore.transactions t=2024-09-14T03:46:49.871071208Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:46:50.131 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=sqlstore.transactions t=2024-09-14T03:46:49.882013782Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-14T03:46:50.131 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=sqlstore.transactions t=2024-09-14T03:46:49.894139961Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:46:50.131 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=provisioning.dashboard t=2024-09-14T03:46:49.961325311Z level=info msg="finished to provision dashboards" 2024-09-14T03:46:50.131 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=grafana-apiserver t=2024-09-14T03:46:50.130043843Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-14T03:46:50.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:46:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=grafana-apiserver t=2024-09-14T03:46:50.130803145Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-14T03:46:50.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:46:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:46:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:46:50.838 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1556 (ObjNum 807 snap 267 seq_num 807) dirty exists 2024-09-14T03:46:50.838 INFO:tasks.rados.rados.0.smithi110.stdout:2644: done (1 left) 2024-09-14T03:46:50.838 INFO:tasks.rados.rados.0.smithi110.stdout:2645: done (0 left) 2024-09-14T03:46:50.838 INFO:tasks.rados.rados.0.smithi110.stdout:2646: rollback oid 38 current snap is 271 2024-09-14T03:46:50.838 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 251 2024-09-14T03:46:50.838 INFO:tasks.rados.rados.0.smithi110.stdout:2647: snap_remove snap 257 2024-09-14T03:46:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[88262]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T03:46:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[88262]: pgmap v542: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 214 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 5 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T03:46:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[86423]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T03:46:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[86423]: pgmap v542: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 214 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 5 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T03:46:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:51.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:50 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:50 smithi195 ceph-mon[82388]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T03:46:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:50 smithi195 ceph-mon[82388]: pgmap v542: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 214 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 5 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T03:46:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:50 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:50 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:50 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:46:51.838 INFO:tasks.rados.rados.0.smithi110.stdout:2647: done (1 left) 2024-09-14T03:46:51.839 INFO:tasks.rados.rados.0.smithi110.stdout:2648: copy_from oid 41 from oid 7 current snap is 271 2024-09-14T03:46:51.839 INFO:tasks.rados.rados.0.smithi110.stdout:2649: read oid 3 snap -1 2024-09-14T03:46:51.839 INFO:tasks.rados.rados.0.smithi110.stdout:2649: expect deleted 2024-09-14T03:46:51.839 INFO:tasks.rados.rados.0.smithi110.stdout:2650: snap_create 2024-09-14T03:46:51.839 INFO:tasks.rados.rados.0.smithi110.stdout:2646: finishing rollback tid 0 to smithi11075562-38 2024-09-14T03:46:51.839 INFO:tasks.rados.rados.0.smithi110.stdout:2646: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:46:51.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1598 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:46:51.863 INFO:tasks.rados.rados.0.smithi110.stdout:2648: finishing copy_from racing read to smithi11075562-41 2024-09-14T03:46:51.864 INFO:tasks.rados.rados.0.smithi110.stdout:2648: finishing copy_from to smithi11075562-41 2024-09-14T03:46:51.864 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1596 (ObjNum 816 snap 270 seq_num 816) dirty exists 2024-09-14T03:46:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:46:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:46:51] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-14T03:46:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:51 smithi110 ceph-mon[88262]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:46:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:51 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:46:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:51 smithi110 ceph-mon[88262]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T03:46:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:51 smithi110 ceph-mon[86423]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:46:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:51 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:46:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:51 smithi110 ceph-mon[86423]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T03:46:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:51 smithi195 ceph-mon[82388]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:46:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:51 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:46:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:51 smithi195 ceph-mon[82388]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T03:46:52.839 INFO:tasks.rados.rados.0.smithi110.stdout:2646: done (3 left) 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2648: done (2 left) 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2649: done (1 left) 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2650: done (0 left) 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2651: rmattr oid 7 current snap is 272 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2652: read oid 45 snap -1 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2652: expect deleted 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2653: read oid 15 snap -1 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2653: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2654: rollback oid 47 current snap is 272 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 47 to 264 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2655: rmattr oid 17 current snap is 272 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:2655: done (4 left) 2024-09-14T03:46:52.840 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 272 2024-09-14T03:46:52.841 INFO:tasks.rados.rados.0.smithi110.stdout:2656: seq_num 818 ranges {0=507904,507904=704512,1212416=499712} 2024-09-14T03:46:52.842 INFO:tasks.rados.rados.0.smithi110.stdout:2656: writing smithi11075562-44 from 0 to 507904 tid 1 2024-09-14T03:46:52.846 INFO:tasks.rados.rados.0.smithi110.stdout:2656: writing smithi11075562-44 from 507904 to 1212416 tid 2 2024-09-14T03:46:52.849 INFO:tasks.rados.rados.0.smithi110.stdout:2656: writing smithi11075562-44 from 1212416 to 1712128 tid 3 2024-09-14T03:46:52.849 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 272 2024-09-14T03:46:52.850 INFO:tasks.rados.rados.0.smithi110.stdout:2657: seq_num 819 ranges {2932736=524288,3457024=589824,4046848=614400} 2024-09-14T03:46:52.868 INFO:tasks.rados.rados.0.smithi110.stdout:2657: writing smithi11075562-9 from 2932736 to 3457024 tid 1 2024-09-14T03:46:52.871 INFO:tasks.rados.rados.0.smithi110.stdout:2657: writing smithi11075562-9 from 3457024 to 4046848 tid 2 2024-09-14T03:46:52.875 INFO:tasks.rados.rados.0.smithi110.stdout:2657: writing smithi11075562-9 from 4046848 to 4661248 tid 3 2024-09-14T03:46:52.875 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 272 2024-09-14T03:46:52.875 INFO:tasks.rados.rados.0.smithi110.stdout:2658: seq_num 820 ranges {7503872=540672,8044544=475136,8519680=598016,9117696=131072} 2024-09-14T03:46:52.918 INFO:tasks.rados.rados.0.smithi110.stdout:2658: writing smithi11075562-1 from 7503872 to 8044544 tid 1 2024-09-14T03:46:52.920 INFO:tasks.rados.rados.0.smithi110.stdout:2658: writing smithi11075562-1 from 8044544 to 8519680 tid 2 2024-09-14T03:46:52.924 INFO:tasks.rados.rados.0.smithi110.stdout:2658: writing smithi11075562-1 from 8519680 to 9117696 tid 3 2024-09-14T03:46:52.925 INFO:tasks.rados.rados.0.smithi110.stdout:2658: writing smithi11075562-1 from 9117696 to 9248768 tid 4 2024-09-14T03:46:52.925 INFO:tasks.rados.rados.0.smithi110.stdout:2659: delete oid 34 current snap is 272 2024-09-14T03:46:52.925 INFO:tasks.rados.rados.0.smithi110.stdout:2653: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:46:52.926 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1599 (ObjNum 816 snap 270 seq_num 816) dirty exists 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2654: finishing rollback tid 1 to smithi11075562-47 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1714 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2656: finishing write tid 1 to smithi11075562-44 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2656: finishing write tid 2 to smithi11075562-44 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2656: finishing write tid 3 to smithi11075562-44 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2656: finishing write tid 4 to smithi11075562-44 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2656: finishing write tid 5 to smithi11075562-44 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 418 (ObjNum 818 snap 272 seq_num 818) dirty exists 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2656: left oid 44 (ObjNum 818 snap 272 seq_num 818) 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2657: finishing write tid 1 to smithi11075562-9 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2657: finishing write tid 2 to smithi11075562-9 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2657: finishing write tid 3 to smithi11075562-9 2024-09-14T03:46:52.927 INFO:tasks.rados.rados.0.smithi110.stdout:2657: finishing write tid 4 to smithi11075562-9 2024-09-14T03:46:52.928 INFO:tasks.rados.rados.0.smithi110.stdout:2657: finishing write tid 5 to smithi11075562-9 2024-09-14T03:46:52.928 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1595 (ObjNum 819 snap 272 seq_num 819) dirty exists 2024-09-14T03:46:52.928 INFO:tasks.rados.rados.0.smithi110.stdout:2657: left oid 9 (ObjNum 819 snap 272 seq_num 819) 2024-09-14T03:46:52.928 INFO:tasks.rados.rados.0.smithi110.stdout:2658: finishing write tid 1 to smithi11075562-1 2024-09-14T03:46:52.928 INFO:tasks.rados.rados.0.smithi110.stdout:2658: finishing write tid 2 to smithi11075562-1 2024-09-14T03:46:52.928 INFO:tasks.rados.rados.0.smithi110.stdout:2651: done (7 left) 2024-09-14T03:46:52.928 INFO:tasks.rados.rados.0.smithi110.stdout:2652: done (6 left) 2024-09-14T03:46:52.929 INFO:tasks.rados.rados.0.smithi110.stdout:2653: done (5 left) 2024-09-14T03:46:52.929 INFO:tasks.rados.rados.0.smithi110.stdout:2654: done (4 left) 2024-09-14T03:46:52.929 INFO:tasks.rados.rados.0.smithi110.stdout:2656: done (3 left) 2024-09-14T03:46:52.929 INFO:tasks.rados.rados.0.smithi110.stdout:2657: done (2 left) 2024-09-14T03:46:52.929 INFO:tasks.rados.rados.0.smithi110.stdout:2659: done (1 left) 2024-09-14T03:46:52.929 INFO:tasks.rados.rados.0.smithi110.stdout:2660: snap_create 2024-09-14T03:46:52.929 INFO:tasks.rados.rados.0.smithi110.stdout:2658: finishing write tid 3 to smithi11075562-1 2024-09-14T03:46:52.930 INFO:tasks.rados.rados.0.smithi110.stdout:2658: finishing write tid 4 to smithi11075562-1 2024-09-14T03:46:52.930 INFO:tasks.rados.rados.0.smithi110.stdout:2658: finishing write tid 5 to smithi11075562-1 2024-09-14T03:46:52.931 INFO:tasks.rados.rados.0.smithi110.stdout:2658: finishing write tid 6 to smithi11075562-1 2024-09-14T03:46:52.932 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1572 (ObjNum 820 snap 272 seq_num 820) dirty exists 2024-09-14T03:46:52.932 INFO:tasks.rados.rados.0.smithi110.stdout:2658: left oid 1 (ObjNum 820 snap 272 seq_num 820) 2024-09-14T03:46:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:52 smithi110 ceph-mon[88262]: pgmap v544: 121 pgs: 121 active+clean; 218 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 12 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T03:46:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:52 smithi110 ceph-mon[88262]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T03:46:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:52 smithi110 ceph-mon[86423]: pgmap v544: 121 pgs: 121 active+clean; 218 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 12 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T03:46:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:52 smithi110 ceph-mon[86423]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T03:46:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:52 smithi195 ceph-mon[82388]: pgmap v544: 121 pgs: 121 active+clean; 218 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 12 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T03:46:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:52 smithi195 ceph-mon[82388]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T03:46:53.760 INFO:tasks.rados.rados.0.smithi110.stdout:2658: done (1 left) 2024-09-14T03:46:53.760 INFO:tasks.rados.rados.0.smithi110.stdout:2660: done (0 left) 2024-09-14T03:46:53.760 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 273 2024-09-14T03:46:53.760 INFO:tasks.rados.rados.0.smithi110.stdout:2661: seq_num 822 ranges {0=573440,573440=696320,1269760=507904} 2024-09-14T03:46:53.763 INFO:tasks.rados.rados.0.smithi110.stdout:2661: writing smithi11075562-48 from 0 to 573440 tid 1 2024-09-14T03:46:53.767 INFO:tasks.rados.rados.0.smithi110.stdout:2661: writing smithi11075562-48 from 573440 to 1269760 tid 2 2024-09-14T03:46:53.770 INFO:tasks.rados.rados.0.smithi110.stdout:2661: writing smithi11075562-48 from 1269760 to 1777664 tid 3 2024-09-14T03:46:53.770 INFO:tasks.rados.rados.0.smithi110.stdout:2662: delete oid 9 current snap is 273 2024-09-14T03:46:53.770 INFO:tasks.rados.rados.0.smithi110.stdout:2661: finishing write tid 1 to smithi11075562-48 2024-09-14T03:46:53.770 INFO:tasks.rados.rados.0.smithi110.stdout:2661: finishing write tid 2 to smithi11075562-48 2024-09-14T03:46:53.774 INFO:tasks.rados.rados.0.smithi110.stdout:2661: finishing write tid 3 to smithi11075562-48 2024-09-14T03:46:53.774 INFO:tasks.rados.rados.0.smithi110.stdout:2661: finishing write tid 4 to smithi11075562-48 2024-09-14T03:46:53.775 INFO:tasks.rados.rados.0.smithi110.stdout:2662: done (1 left) 2024-09-14T03:46:53.775 INFO:tasks.rados.rados.0.smithi110.stdout:2663: read oid 25 snap -1 2024-09-14T03:46:53.775 INFO:tasks.rados.rados.0.smithi110.stdout:2663: expect (ObjNum 642 snap 199 seq_num 642) 2024-09-14T03:46:53.775 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 273 2024-09-14T03:46:53.775 INFO:tasks.rados.rados.0.smithi110.stdout:2664: seq_num 824 ranges {0=606208,606208=458752,1064960=573440,1638400=172032} 2024-09-14T03:46:53.778 INFO:tasks.rados.rados.0.smithi110.stdout:2664: writing smithi11075562-38 from 0 to 606208 tid 1 2024-09-14T03:46:53.781 INFO:tasks.rados.rados.0.smithi110.stdout:2664: writing smithi11075562-38 from 606208 to 1064960 tid 2 2024-09-14T03:46:53.784 INFO:tasks.rados.rados.0.smithi110.stdout:2664: writing smithi11075562-38 from 1064960 to 1638400 tid 3 2024-09-14T03:46:53.786 INFO:tasks.rados.rados.0.smithi110.stdout:2664: writing smithi11075562-38 from 1638400 to 1810432 tid 4 2024-09-14T03:46:53.786 INFO:tasks.rados.rados.0.smithi110.stdout:2665: snap_remove snap 263 2024-09-14T03:46:54.101 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:53 smithi195 ceph-mon[82388]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T03:46:54.101 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:53 smithi195 ceph-mon[82388]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T03:46:54.101 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:53 smithi195 ceph-mon[82388]: pgmap v548: 121 pgs: 121 active+clean; 224 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.3 MiB/s wr, 15 op/s 2024-09-14T03:46:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:53 smithi110 ceph-mon[88262]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T03:46:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:53 smithi110 ceph-mon[88262]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T03:46:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:53 smithi110 ceph-mon[88262]: pgmap v548: 121 pgs: 121 active+clean; 224 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.3 MiB/s wr, 15 op/s 2024-09-14T03:46:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:53 smithi110 ceph-mon[86423]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T03:46:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:53 smithi110 ceph-mon[86423]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T03:46:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:53 smithi110 ceph-mon[86423]: pgmap v548: 121 pgs: 121 active+clean; 224 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.3 MiB/s wr, 15 op/s 2024-09-14T03:46:54.765 INFO:tasks.rados.rados.0.smithi110.stdout:2665: done (3 left) 2024-09-14T03:46:54.765 INFO:tasks.rados.rados.0.smithi110.stdout:2666: read oid 8 snap -1 2024-09-14T03:46:54.765 INFO:tasks.rados.rados.0.smithi110.stdout:2666: expect deleted 2024-09-14T03:46:54.766 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 273 2024-09-14T03:46:54.766 INFO:tasks.rados.rados.0.smithi110.stdout:2667: seq_num 825 ranges {966656=622592,1589248=745472,2334720=458752} 2024-09-14T03:46:54.775 INFO:tasks.rados.rados.0.smithi110.stdout:2667: writing smithi11075562-10 from 966656 to 1589248 tid 1 2024-09-14T03:46:54.780 INFO:tasks.rados.rados.0.smithi110.stdout:2667: writing smithi11075562-10 from 1589248 to 2334720 tid 2 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:2667: writing smithi11075562-10 from 2334720 to 2793472 tid 3 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:2668: snap_create 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:2661: finishing write tid 5 to smithi11075562-48 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1573 (ObjNum 822 snap 273 seq_num 822) dirty exists 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:2661: left oid 48 (ObjNum 822 snap 273 seq_num 822) 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:2664: finishing write tid 1 to smithi11075562-38 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:2664: finishing write tid 2 to smithi11075562-38 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:2664: finishing write tid 3 to smithi11075562-38 2024-09-14T03:46:54.784 INFO:tasks.rados.rados.0.smithi110.stdout:2664: finishing write tid 4 to smithi11075562-38 2024-09-14T03:46:54.785 INFO:tasks.rados.rados.0.smithi110.stdout:2664: finishing write tid 5 to smithi11075562-38 2024-09-14T03:46:54.785 INFO:tasks.rados.rados.0.smithi110.stdout:2664: finishing write tid 6 to smithi11075562-38 2024-09-14T03:46:54.785 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1604 (ObjNum 824 snap 273 seq_num 824) dirty exists 2024-09-14T03:46:54.785 INFO:tasks.rados.rados.0.smithi110.stdout:2664: left oid 38 (ObjNum 824 snap 273 seq_num 824) 2024-09-14T03:46:54.785 INFO:tasks.rados.rados.0.smithi110.stdout:2663: expect (ObjNum 642 snap 199 seq_num 642) 2024-09-14T03:46:55.557 INFO:tasks.rados.rados.0.smithi110.stdout:2667: finishing write tid 1 to smithi11075562-10 2024-09-14T03:46:55.557 INFO:tasks.rados.rados.0.smithi110.stdout:2667: finishing write tid 2 to smithi11075562-10 2024-09-14T03:46:55.557 INFO:tasks.rados.rados.0.smithi110.stdout:2667: finishing write tid 3 to smithi11075562-10 2024-09-14T03:46:55.557 INFO:tasks.rados.rados.0.smithi110.stdout:2667: finishing write tid 4 to smithi11075562-10 2024-09-14T03:46:55.558 INFO:tasks.rados.rados.0.smithi110.stdout:2667: finishing write tid 5 to smithi11075562-10 2024-09-14T03:46:55.558 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1504 (ObjNum 825 snap 273 seq_num 825) dirty exists 2024-09-14T03:46:55.558 INFO:tasks.rados.rados.0.smithi110.stdout:2667: left oid 10 (ObjNum 825 snap 273 seq_num 825) 2024-09-14T03:46:55.767 INFO:tasks.rados.rados.0.smithi110.stdout:2661: done (5 left) 2024-09-14T03:46:55.767 INFO:tasks.rados.rados.0.smithi110.stdout:2663: done (4 left) 2024-09-14T03:46:55.768 INFO:tasks.rados.rados.0.smithi110.stdout:2664: done (3 left) 2024-09-14T03:46:55.768 INFO:tasks.rados.rados.0.smithi110.stdout:2666: done (2 left) 2024-09-14T03:46:55.768 INFO:tasks.rados.rados.0.smithi110.stdout:2667: done (1 left) 2024-09-14T03:46:55.768 INFO:tasks.rados.rados.0.smithi110.stdout:2668: done (0 left) 2024-09-14T03:46:55.768 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 274 2024-09-14T03:46:55.768 INFO:tasks.rados.rados.0.smithi110.stdout:2669: seq_num 826 ranges {5292032=638976,5931008=630784,6561792=573440} 2024-09-14T03:46:55.798 INFO:tasks.rados.rados.0.smithi110.stdout:2669: writing smithi11075562-12 from 5292032 to 5931008 tid 1 2024-09-14T03:46:55.802 INFO:tasks.rados.rados.0.smithi110.stdout:2669: writing smithi11075562-12 from 5931008 to 6561792 tid 2 2024-09-14T03:46:55.806 INFO:tasks.rados.rados.0.smithi110.stdout:2669: writing smithi11075562-12 from 6561792 to 7135232 tid 3 2024-09-14T03:46:55.806 INFO:tasks.rados.rados.0.smithi110.stdout:2670: rollback oid 49 current snap is 274 2024-09-14T03:46:55.806 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 259 2024-09-14T03:46:55.806 INFO:tasks.rados.rados.0.smithi110.stdout:2671: copy_from oid 20 from oid 40 current snap is 274 2024-09-14T03:46:55.806 INFO:tasks.rados.rados.0.smithi110.stdout:2672: rollback oid 8 current snap is 274 2024-09-14T03:46:55.806 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 8 to 259 2024-09-14T03:46:55.806 INFO:tasks.rados.rados.0.smithi110.stdout:2673: delete oid 31 current snap is 274 2024-09-14T03:46:55.807 INFO:tasks.rados.rados.0.smithi110.stdout:2669: finishing write tid 1 to smithi11075562-12 2024-09-14T03:46:55.808 INFO:tasks.rados.rados.0.smithi110.stdout:2672: finishing rollback tid 1 to smithi11075562-8 2024-09-14T03:46:55.809 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 0 (ObjNum 33 snap 0 seq_num 43181432) dirty dne 2024-09-14T03:46:55.810 INFO:tasks.rados.rados.0.smithi110.stdout:2669: finishing write tid 2 to smithi11075562-12 2024-09-14T03:46:55.811 INFO:tasks.rados.rados.0.smithi110.stdout:2671: finishing copy_from to smithi11075562-20 2024-09-14T03:46:55.811 INFO:tasks.rados.rados.0.smithi110.stdout:2671: got expected ENOENT (src dne) 2024-09-14T03:46:55.811 INFO:tasks.rados.rados.0.smithi110.stdout:2671: finishing copy_from racing read to smithi11075562-20 2024-09-14T03:46:55.812 INFO:tasks.rados.rados.0.smithi110.stdout:2669: finishing write tid 3 to smithi11075562-12 2024-09-14T03:46:55.812 INFO:tasks.rados.rados.0.smithi110.stdout:2669: finishing write tid 4 to smithi11075562-12 2024-09-14T03:46:55.812 INFO:tasks.rados.rados.0.smithi110.stdout:2670: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:46:55.812 INFO:tasks.rados.rados.0.smithi110.stdout:2670: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:46:55.812 INFO:tasks.rados.rados.0.smithi110.stdout:2670: finishing rollback tid 2 to smithi11075562-49 2024-09-14T03:46:55.812 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1584 (ObjNum 784 snap 257 seq_num 784) dirty exists 2024-09-14T03:46:55.813 INFO:tasks.rados.rados.0.smithi110.stdout:2670: done (4 left) 2024-09-14T03:46:55.813 INFO:tasks.rados.rados.0.smithi110.stdout:2671: done (3 left) 2024-09-14T03:46:55.813 INFO:tasks.rados.rados.0.smithi110.stdout:2672: done (2 left) 2024-09-14T03:46:55.813 INFO:tasks.rados.rados.0.smithi110.stdout:2673: done (1 left) 2024-09-14T03:46:55.813 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 274 2024-09-14T03:46:55.813 INFO:tasks.rados.rados.0.smithi110.stdout:2674: seq_num 828 ranges {1474560=671744,2146304=802816,2949120=401408} 2024-09-14T03:46:55.824 INFO:tasks.rados.rados.0.smithi110.stdout:2674: writing smithi11075562-6 from 1474560 to 2146304 tid 1 2024-09-14T03:46:55.829 INFO:tasks.rados.rados.0.smithi110.stdout:2674: writing smithi11075562-6 from 2146304 to 2949120 tid 2 2024-09-14T03:46:55.832 INFO:tasks.rados.rados.0.smithi110.stdout:2674: writing smithi11075562-6 from 2949120 to 3350528 tid 3 2024-09-14T03:46:55.832 INFO:tasks.rados.rados.0.smithi110.stdout:2675: setattr oid 20 current snap is 274 2024-09-14T03:46:55.832 INFO:tasks.rados.rados.0.smithi110.stdout:2669: finishing write tid 5 to smithi11075562-12 2024-09-14T03:46:55.832 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1840 (ObjNum 826 snap 274 seq_num 826) dirty exists 2024-09-14T03:46:55.832 INFO:tasks.rados.rados.0.smithi110.stdout:2669: left oid 12 (ObjNum 826 snap 274 seq_num 826) 2024-09-14T03:46:55.832 INFO:tasks.rados.rados.0.smithi110.stdout:2674: finishing write tid 1 to smithi11075562-6 2024-09-14T03:46:55.832 INFO:tasks.rados.rados.0.smithi110.stdout:2669: done (2 left) 2024-09-14T03:46:55.832 INFO:tasks.rados.rados.0.smithi110.stdout:2676: read oid 33 snap -1 2024-09-14T03:46:55.833 INFO:tasks.rados.rados.0.smithi110.stdout:2676: expect (ObjNum 701 snap 224 seq_num 701) 2024-09-14T03:46:55.833 INFO:tasks.rados.rados.0.smithi110.stdout:2677: copy_from oid 35 from oid 5 current snap is 274 2024-09-14T03:46:55.833 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 274 2024-09-14T03:46:55.833 INFO:tasks.rados.rados.0.smithi110.stdout:2678: seq_num 829 ranges {9699328=688128,10387456=729088,11116544=475136} 2024-09-14T03:46:55.888 INFO:tasks.rados.rados.0.smithi110.stdout:2678: writing smithi11075562-26 from 9699328 to 10387456 tid 1 2024-09-14T03:46:55.892 INFO:tasks.rados.rados.0.smithi110.stdout:2678: writing smithi11075562-26 from 10387456 to 11116544 tid 2 2024-09-14T03:46:55.895 INFO:tasks.rados.rados.0.smithi110.stdout:2678: writing smithi11075562-26 from 11116544 to 11591680 tid 3 2024-09-14T03:46:55.895 INFO:tasks.rados.rados.0.smithi110.stdout:2679: read oid 17 snap -1 2024-09-14T03:46:55.895 INFO:tasks.rados.rados.0.smithi110.stdout:2679: expect (ObjNum 754 snap 244 seq_num 754) 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:2680: delete oid 16 current snap is 274 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:2674: finishing write tid 2 to smithi11075562-6 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:2674: finishing write tid 3 to smithi11075562-6 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:2674: finishing write tid 4 to smithi11075562-6 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:2674: finishing write tid 5 to smithi11075562-6 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1506 (ObjNum 828 snap 274 seq_num 828) dirty exists 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:2674: left oid 6 (ObjNum 828 snap 274 seq_num 828) 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1597 (ObjNum 33 snap 0 seq_num 65137216) dirty exists 2024-09-14T03:46:55.896 INFO:tasks.rados.rados.0.smithi110.stdout:2676: expect (ObjNum 701 snap 224 seq_num 701) 2024-09-14T03:46:56.099 INFO:tasks.rados.rados.0.smithi110.stdout:2677: finishing copy_from racing read to smithi11075562-35 2024-09-14T03:46:56.099 INFO:tasks.rados.rados.0.smithi110.stdout:2677: finishing copy_from to smithi11075562-35 2024-09-14T03:46:56.099 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1808 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:46:56.099 INFO:tasks.rados.rados.0.smithi110.stdout:2678: finishing write tid 1 to smithi11075562-26 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2674: done (6 left) 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2675: done (5 left) 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2676: done (4 left) 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2677: done (3 left) 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2680: done (2 left) 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2681: read oid 36 snap -1 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2681: expect deleted 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 274 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2678: finishing write tid 2 to smithi11075562-26 2024-09-14T03:46:56.100 INFO:tasks.rados.rados.0.smithi110.stdout:2682: seq_num 831 ranges {0=720896,720896=499712,1220608=704512} 2024-09-14T03:46:56.103 INFO:tasks.rados.rados.0.smithi110.stdout:2682: writing smithi11075562-40 from 0 to 720896 tid 1 2024-09-14T03:46:56.105 INFO:tasks.rados.rados.0.smithi110.stdout:2682: writing smithi11075562-40 from 720896 to 1220608 tid 2 2024-09-14T03:46:56.110 INFO:tasks.rados.rados.0.smithi110.stdout:2682: writing smithi11075562-40 from 1220608 to 1925120 tid 3 2024-09-14T03:46:56.110 INFO:tasks.rados.rados.0.smithi110.stdout:2683: snap_remove snap 272 2024-09-14T03:46:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:55 smithi110 ceph-mon[88262]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T03:46:56.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:55 smithi110 ceph-mon[86423]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T03:46:56.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:55 smithi195 ceph-mon[82388]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T03:46:56.953 INFO:tasks.rados.rados.0.smithi110.stdout:2683: done (4 left) 2024-09-14T03:46:56.953 INFO:tasks.rados.rados.0.smithi110.stdout:2684: copy_from oid 32 from oid 39 current snap is 274 2024-09-14T03:46:56.953 INFO:tasks.rados.rados.0.smithi110.stdout:2685: read oid 8 snap 270 2024-09-14T03:46:56.953 INFO:tasks.rados.rados.0.smithi110.stdout:2685: expect deleted 2024-09-14T03:46:56.953 INFO:tasks.rados.rados.0.smithi110.stdout:2686: copy_from oid 5 from oid 4 current snap is 274 2024-09-14T03:46:56.953 INFO:tasks.rados.rados.0.smithi110.stdout:2678: finishing write tid 3 to smithi11075562-26 2024-09-14T03:46:56.953 INFO:tasks.rados.rados.0.smithi110.stdout:2678: finishing write tid 4 to smithi11075562-26 2024-09-14T03:46:56.954 INFO:tasks.rados.rados.0.smithi110.stdout:2678: finishing write tid 5 to smithi11075562-26 2024-09-14T03:46:56.954 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1450 (ObjNum 829 snap 274 seq_num 829) dirty exists 2024-09-14T03:46:56.954 INFO:tasks.rados.rados.0.smithi110.stdout:2678: left oid 26 (ObjNum 829 snap 274 seq_num 829) 2024-09-14T03:46:56.954 INFO:tasks.rados.rados.0.smithi110.stdout:2679: expect (ObjNum 754 snap 244 seq_num 754) 2024-09-14T03:46:56.974 INFO:tasks.rados.rados.0.smithi110.stdout:2682: finishing write tid 1 to smithi11075562-40 2024-09-14T03:46:56.974 INFO:tasks.rados.rados.0.smithi110.stdout:2682: finishing write tid 2 to smithi11075562-40 2024-09-14T03:46:56.974 INFO:tasks.rados.rados.0.smithi110.stdout:2678: done (6 left) 2024-09-14T03:46:56.975 INFO:tasks.rados.rados.0.smithi110.stdout:2679: done (5 left) 2024-09-14T03:46:56.975 INFO:tasks.rados.rados.0.smithi110.stdout:2681: done (4 left) 2024-09-14T03:46:56.975 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 274 2024-09-14T03:46:56.975 INFO:tasks.rados.rados.0.smithi110.stdout:2682: finishing write tid 3 to smithi11075562-40 2024-09-14T03:46:56.975 INFO:tasks.rados.rados.0.smithi110.stdout:2687: seq_num 832 ranges {2793472=737280,3530752=786432,4317184=417792} 2024-09-14T03:46:56.994 INFO:tasks.rados.rados.0.smithi110.stdout:2687: writing smithi11075562-10 from 2793472 to 3530752 tid 1 2024-09-14T03:46:56.999 INFO:tasks.rados.rados.0.smithi110.stdout:2687: writing smithi11075562-10 from 3530752 to 4317184 tid 2 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:2687: writing smithi11075562-10 from 4317184 to 4734976 tid 3 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:2688: delete oid 18 current snap is 274 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:2682: finishing write tid 4 to smithi11075562-40 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:2682: finishing write tid 5 to smithi11075562-40 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1511 (ObjNum 831 snap 274 seq_num 831) dirty exists 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:2682: left oid 40 (ObjNum 831 snap 274 seq_num 831) 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:2686: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:2686: finishing copy_from to smithi11075562-5 2024-09-14T03:46:57.003 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1575 (ObjNum 796 snap 264 seq_num 796) dirty exists 2024-09-14T03:46:57.004 INFO:tasks.rados.rados.0.smithi110.stdout:2687: finishing write tid 1 to smithi11075562-10 2024-09-14T03:46:57.005 INFO:tasks.rados.rados.0.smithi110.stdout:2682: done (5 left) 2024-09-14T03:46:57.005 INFO:tasks.rados.rados.0.smithi110.stdout:2685: done (4 left) 2024-09-14T03:46:57.005 INFO:tasks.rados.rados.0.smithi110.stdout:2686: done (3 left) 2024-09-14T03:46:57.005 INFO:tasks.rados.rados.0.smithi110.stdout:2688: done (2 left) 2024-09-14T03:46:57.005 INFO:tasks.rados.rados.0.smithi110.stdout:2689: copy_from oid 41 from oid 31 current snap is 274 2024-09-14T03:46:57.006 INFO:tasks.rados.rados.0.smithi110.stdout:2690: snap_remove snap 259 2024-09-14T03:46:57.026 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:56 smithi195 ceph-mon[82388]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T03:46:57.026 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:56 smithi195 ceph-mon[82388]: pgmap v551: 121 pgs: 121 active+clean; 222 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-14T03:46:57.026 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:56 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.026 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:56 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.026 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:56 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.026 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:56 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[88262]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T03:46:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[88262]: pgmap v551: 121 pgs: 121 active+clean; 222 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-14T03:46:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[86423]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T03:46:57.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[86423]: pgmap v551: 121 pgs: 121 active+clean; 222 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-14T03:46:57.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:56 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:57.960 INFO:tasks.rados.rados.0.smithi110.stdout:2690: done (3 left) 2024-09-14T03:46:57.960 INFO:tasks.rados.rados.0.smithi110.stdout:2691: read oid 6 snap 266 2024-09-14T03:46:57.960 INFO:tasks.rados.rados.0.smithi110.stdout:2691: expect deleted 2024-09-14T03:46:57.960 INFO:tasks.rados.rados.0.smithi110.stdout:2692: rollback oid 29 current snap is 274 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 29 to 270 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2689: finishing copy_from to smithi11075562-41 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2689: got expected ENOENT (src dne) 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2689: finishing copy_from racing read to smithi11075562-41 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2687: finishing write tid 2 to smithi11075562-10 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2687: finishing write tid 3 to smithi11075562-10 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2687: finishing write tid 4 to smithi11075562-10 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2687: finishing write tid 5 to smithi11075562-10 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1508 (ObjNum 832 snap 274 seq_num 832) dirty exists 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2687: left oid 10 (ObjNum 832 snap 274 seq_num 832) 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2684: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:2684: finishing copy_from to smithi11075562-32 2024-09-14T03:46:57.961 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1514 (ObjNum 789 snap 259 seq_num 789) dirty exists 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2684: done (4 left) 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2687: done (3 left) 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2689: done (2 left) 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2693: read oid 5 snap -1 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2693: expect (ObjNum 796 snap 264 seq_num 796) 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2694: read oid 3 snap 262 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2694: expect deleted 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2695: read oid 22 snap -1 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2695: expect (ObjNum 794 snap 263 seq_num 794) 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2696: read oid 48 snap -1 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2696: expect (ObjNum 822 snap 273 seq_num 822) 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2697: read oid 45 snap -1 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2697: expect deleted 2024-09-14T03:46:57.962 INFO:tasks.rados.rados.0.smithi110.stdout:2698: snap_create 2024-09-14T03:46:57.979 INFO:tasks.rados.rados.0.smithi110.stdout:2692: finishing rollback tid 0 to smithi11075562-29 2024-09-14T03:46:57.979 INFO:tasks.rados.rados.0.smithi110.stdout:2696: expect (ObjNum 822 snap 273 seq_num 822) 2024-09-14T03:46:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:2692: finishing rollback tid 1 to smithi11075562-29 2024-09-14T03:46:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:2692: finishing rollback tid 2 to smithi11075562-29 2024-09-14T03:46:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1607 (ObjNum 653 snap 202 seq_num 653) dirty exists 2024-09-14T03:46:58.029 INFO:tasks.rados.rados.0.smithi110.stdout:2693: expect (ObjNum 796 snap 264 seq_num 796) 2024-09-14T03:46:58.137 INFO:tasks.rados.rados.0.smithi110.stdout:2695: expect (ObjNum 794 snap 263 seq_num 794) 2024-09-14T03:46:58.143 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:57 smithi195 ceph-mon[82388]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T03:46:58.143 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:58.143 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:58.143 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:57 smithi195 ceph-mon[82388]: pgmap v553: 121 pgs: 121 active+clean; 236 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.4 MiB/s wr, 13 op/s 2024-09-14T03:46:58.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:57 smithi110 ceph-mon[88262]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T03:46:58.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:58.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:58.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:57 smithi110 ceph-mon[88262]: pgmap v553: 121 pgs: 121 active+clean; 236 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.4 MiB/s wr, 13 op/s 2024-09-14T03:46:58.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:57 smithi110 ceph-mon[86423]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T03:46:58.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:46:58.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:46:58.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:57 smithi110 ceph-mon[86423]: pgmap v553: 121 pgs: 121 active+clean; 236 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.4 MiB/s wr, 13 op/s 2024-09-14T03:46:58.762 INFO:tasks.rados.rados.0.smithi110.stdout:2691: done (7 left) 2024-09-14T03:46:58.762 INFO:tasks.rados.rados.0.smithi110.stdout:2692: done (6 left) 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:2693: done (5 left) 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:2694: done (4 left) 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:2695: done (3 left) 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:2696: done (2 left) 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:2697: done (1 left) 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:2698: done (0 left) 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:2699: copy_from oid 38 from oid 17 current snap is 275 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 275 2024-09-14T03:46:58.763 INFO:tasks.rados.rados.0.smithi110.stdout:2700: seq_num 834 ranges {0=778240,778240=557056,1335296=548864,1884160=98304} 2024-09-14T03:46:58.767 INFO:tasks.rados.rados.0.smithi110.stdout:2700: writing smithi11075562-11 from 0 to 778240 tid 1 2024-09-14T03:46:58.771 INFO:tasks.rados.rados.0.smithi110.stdout:2700: writing smithi11075562-11 from 778240 to 1335296 tid 2 2024-09-14T03:46:58.774 INFO:tasks.rados.rados.0.smithi110.stdout:2700: writing smithi11075562-11 from 1335296 to 1884160 tid 3 2024-09-14T03:46:58.775 INFO:tasks.rados.rados.0.smithi110.stdout:2700: writing smithi11075562-11 from 1884160 to 1982464 tid 4 2024-09-14T03:46:58.775 INFO:tasks.rados.rados.0.smithi110.stdout:2701: delete oid 47 current snap is 275 2024-09-14T03:46:58.775 INFO:tasks.rados.rados.0.smithi110.stdout:2699: finishing copy_from racing read to smithi11075562-38 2024-09-14T03:46:58.776 INFO:tasks.rados.rados.0.smithi110.stdout:2699: finishing copy_from to smithi11075562-38 2024-09-14T03:46:58.776 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1491 (ObjNum 754 snap 244 seq_num 754) dirty exists 2024-09-14T03:46:58.776 INFO:tasks.rados.rados.0.smithi110.stdout:2700: finishing write tid 1 to smithi11075562-11 2024-09-14T03:46:58.776 INFO:tasks.rados.rados.0.smithi110.stdout:2700: finishing write tid 2 to smithi11075562-11 2024-09-14T03:46:58.778 INFO:tasks.rados.rados.0.smithi110.stdout:2699: done (2 left) 2024-09-14T03:46:58.778 INFO:tasks.rados.rados.0.smithi110.stdout:2701: done (1 left) 2024-09-14T03:46:58.778 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 275 2024-09-14T03:46:58.778 INFO:tasks.rados.rados.0.smithi110.stdout:2702: seq_num 836 ranges {4259840=409600,4669440=720896,5390336=516096,5906432=368640} 2024-09-14T03:46:58.803 INFO:tasks.rados.rados.0.smithi110.stdout:2702: writing smithi11075562-42 from 4259840 to 4669440 tid 1 2024-09-14T03:46:58.807 INFO:tasks.rados.rados.0.smithi110.stdout:2702: writing smithi11075562-42 from 4669440 to 5390336 tid 2 2024-09-14T03:46:58.810 INFO:tasks.rados.rados.0.smithi110.stdout:2702: writing smithi11075562-42 from 5390336 to 5906432 tid 3 2024-09-14T03:46:58.813 INFO:tasks.rados.rados.0.smithi110.stdout:2702: writing smithi11075562-42 from 5906432 to 6275072 tid 4 2024-09-14T03:46:58.813 INFO:tasks.rados.rados.0.smithi110.stdout:2700: finishing write tid 3 to smithi11075562-11 2024-09-14T03:46:58.813 INFO:tasks.rados.rados.0.smithi110.stdout:2700: finishing write tid 4 to smithi11075562-11 2024-09-14T03:46:58.813 INFO:tasks.rados.rados.0.smithi110.stdout:2703: delete oid 25 current snap is 275 2024-09-14T03:46:58.814 INFO:tasks.rados.rados.0.smithi110.stdout:2700: finishing write tid 5 to smithi11075562-11 2024-09-14T03:46:58.814 INFO:tasks.rados.rados.0.smithi110.stdout:2700: finishing write tid 6 to smithi11075562-11 2024-09-14T03:46:58.814 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1578 (ObjNum 834 snap 275 seq_num 834) dirty exists 2024-09-14T03:46:58.814 INFO:tasks.rados.rados.0.smithi110.stdout:2700: left oid 11 (ObjNum 834 snap 275 seq_num 834) 2024-09-14T03:46:58.814 INFO:tasks.rados.rados.0.smithi110.stdout:2702: finishing write tid 1 to smithi11075562-42 2024-09-14T03:46:58.814 INFO:tasks.rados.rados.0.smithi110.stdout:2702: finishing write tid 2 to smithi11075562-42 2024-09-14T03:46:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:2702: finishing write tid 3 to smithi11075562-42 2024-09-14T03:46:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:2702: finishing write tid 4 to smithi11075562-42 2024-09-14T03:46:58.817 INFO:tasks.rados.rados.0.smithi110.stdout:2700: done (2 left) 2024-09-14T03:46:58.817 INFO:tasks.rados.rados.0.smithi110.stdout:2703: done (1 left) 2024-09-14T03:46:58.817 INFO:tasks.rados.rados.0.smithi110.stdout:2704: rollback oid 38 current snap is 275 2024-09-14T03:46:58.817 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 273 2024-09-14T03:46:58.818 INFO:tasks.rados.rados.0.smithi110.stdout:2702: finishing write tid 5 to smithi11075562-42 2024-09-14T03:46:58.818 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 275 2024-09-14T03:46:58.818 INFO:tasks.rados.rados.0.smithi110.stdout:2705: seq_num 838 ranges {0=442368,442368=540672,983040=532480,1515520=442368,1957888=90112} 2024-09-14T03:46:58.819 INFO:tasks.rados.rados.0.smithi110.stdout:2705: writing smithi11075562-47 from 0 to 442368 tid 1 2024-09-14T03:46:58.822 INFO:tasks.rados.rados.0.smithi110.stdout:2705: writing smithi11075562-47 from 442368 to 983040 tid 2 2024-09-14T03:46:58.825 INFO:tasks.rados.rados.0.smithi110.stdout:2705: writing smithi11075562-47 from 983040 to 1515520 tid 3 2024-09-14T03:46:58.828 INFO:tasks.rados.rados.0.smithi110.stdout:2705: writing smithi11075562-47 from 1515520 to 1957888 tid 4 2024-09-14T03:46:58.829 INFO:tasks.rados.rados.0.smithi110.stdout:2705: writing smithi11075562-47 from 1957888 to 2048000 tid 5 2024-09-14T03:46:58.829 INFO:tasks.rados.rados.0.smithi110.stdout:2706: read oid 7 snap -1 2024-09-14T03:46:58.829 INFO:tasks.rados.rados.0.smithi110.stdout:2706: expect (ObjNum 816 snap 270 seq_num 816) 2024-09-14T03:46:58.829 INFO:tasks.rados.rados.0.smithi110.stdout:2707: delete oid 14 current snap is 275 2024-09-14T03:46:58.829 INFO:tasks.rados.rados.0.smithi110.stdout:2702: finishing write tid 6 to smithi11075562-42 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1813 (ObjNum 836 snap 275 seq_num 836) dirty exists 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:2702: left oid 42 (ObjNum 836 snap 275 seq_num 836) 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:2704: finishing rollback tid 0 to smithi11075562-38 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:2704: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:2704: finishing rollback tid 2 to smithi11075562-38 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1610 (ObjNum 824 snap 273 seq_num 824) dirty exists 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:2705: finishing write tid 1 to smithi11075562-47 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:2705: finishing write tid 2 to smithi11075562-47 2024-09-14T03:46:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:2705: finishing write tid 3 to smithi11075562-47 2024-09-14T03:46:58.832 INFO:tasks.rados.rados.0.smithi110.stdout:2702: done (4 left) 2024-09-14T03:46:58.832 INFO:tasks.rados.rados.0.smithi110.stdout:2704: done (3 left) 2024-09-14T03:46:58.832 INFO:tasks.rados.rados.0.smithi110.stdout:2707: done (2 left) 2024-09-14T03:46:58.832 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 275 2024-09-14T03:46:58.832 INFO:tasks.rados.rados.0.smithi110.stdout:2708: seq_num 840 ranges {2752512=475136,3227648=712704,3940352=499712,4440064=393216} 2024-09-14T03:46:58.849 INFO:tasks.rados.rados.0.smithi110.stdout:2708: writing smithi11075562-5 from 2752512 to 3227648 tid 1 2024-09-14T03:46:58.853 INFO:tasks.rados.rados.0.smithi110.stdout:2708: writing smithi11075562-5 from 3227648 to 3940352 tid 2 2024-09-14T03:46:58.856 INFO:tasks.rados.rados.0.smithi110.stdout:2708: writing smithi11075562-5 from 3940352 to 4440064 tid 3 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:2708: writing smithi11075562-5 from 4440064 to 4833280 tid 4 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:2709: read oid 2 snap 270 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:2709: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:2710: read oid 12 snap -1 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:2710: expect (ObjNum 826 snap 274 seq_num 826) 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:2711: read oid 38 snap 250 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:2711: expect (ObjNum 664 snap 204 seq_num 664) 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 33 current snap is 275 2024-09-14T03:46:58.859 INFO:tasks.rados.rados.0.smithi110.stdout:2712: seq_num 841 ranges {5087232=491520,5578752=598016,6176768=704512,6881280=303104} 2024-09-14T03:46:58.889 INFO:tasks.rados.rados.0.smithi110.stdout:2712: writing smithi11075562-33 from 5087232 to 5578752 tid 1 2024-09-14T03:46:58.892 INFO:tasks.rados.rados.0.smithi110.stdout:2712: writing smithi11075562-33 from 5578752 to 6176768 tid 2 2024-09-14T03:46:58.896 INFO:tasks.rados.rados.0.smithi110.stdout:2712: writing smithi11075562-33 from 6176768 to 6881280 tid 3 2024-09-14T03:46:58.898 INFO:tasks.rados.rados.0.smithi110.stdout:2712: writing smithi11075562-33 from 6881280 to 7184384 tid 4 2024-09-14T03:46:58.898 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 275 2024-09-14T03:46:58.898 INFO:tasks.rados.rados.0.smithi110.stdout:2705: finishing write tid 4 to smithi11075562-47 2024-09-14T03:46:58.899 INFO:tasks.rados.rados.0.smithi110.stdout:2705: finishing write tid 5 to smithi11075562-47 2024-09-14T03:46:58.899 INFO:tasks.rados.rados.0.smithi110.stdout:2713: seq_num 842 ranges {6275072=507904,6782976=475136,7258112=516096,7774208=499712,8273920=114688} 2024-09-14T03:46:58.937 INFO:tasks.rados.rados.0.smithi110.stdout:2713: writing smithi11075562-42 from 6275072 to 6782976 tid 1 2024-09-14T03:46:58.937 INFO:tasks.rados.rados.0.smithi110.stdout:2713: writing smithi11075562-42 from 6782976 to 7258112 tid 2 2024-09-14T03:46:58.939 INFO:tasks.rados.rados.0.smithi110.stdout:2713: writing smithi11075562-42 from 7258112 to 7774208 tid 3 2024-09-14T03:46:58.943 INFO:tasks.rados.rados.0.smithi110.stdout:2713: writing smithi11075562-42 from 7774208 to 8273920 tid 4 2024-09-14T03:46:58.943 INFO:tasks.rados.rados.0.smithi110.stdout:2713: writing smithi11075562-42 from 8273920 to 8388608 tid 5 2024-09-14T03:46:58.943 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 275 2024-09-14T03:46:58.943 INFO:tasks.rados.rados.0.smithi110.stdout:2705: finishing write tid 6 to smithi11075562-47 2024-09-14T03:46:58.943 INFO:tasks.rados.rados.0.smithi110.stdout:2706: expect (ObjNum 816 snap 270 seq_num 816) 2024-09-14T03:46:59.004 INFO:tasks.rados.rados.0.smithi110.stdout:2705: finishing write tid 7 to smithi11075562-47 2024-09-14T03:46:59.004 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 1859 (ObjNum 838 snap 275 seq_num 838) dirty exists 2024-09-14T03:46:59.005 INFO:tasks.rados.rados.0.smithi110.stdout:2705: left oid 47 (ObjNum 838 snap 275 seq_num 838) 2024-09-14T03:46:59.005 INFO:tasks.rados.rados.0.smithi110.stdout:2708: finishing write tid 1 to smithi11075562-5 2024-09-14T03:46:59.005 INFO:tasks.rados.rados.0.smithi110.stdout:2708: finishing write tid 2 to smithi11075562-5 2024-09-14T03:46:59.005 INFO:tasks.rados.rados.0.smithi110.stdout:2708: finishing write tid 3 to smithi11075562-5 2024-09-14T03:46:59.005 INFO:tasks.rados.rados.0.smithi110.stdout:2708: finishing write tid 4 to smithi11075562-5 2024-09-14T03:46:59.005 INFO:tasks.rados.rados.0.smithi110.stdout:2708: finishing write tid 5 to smithi11075562-5 2024-09-14T03:46:59.005 INFO:tasks.rados.rados.0.smithi110.stdout:2709: expect (ObjNum 649 snap 201 seq_num 649) 2024-09-14T03:46:59.073 INFO:tasks.rados.rados.0.smithi110.stdout:2711: expect (ObjNum 664 snap 204 seq_num 664) 2024-09-14T03:46:59.187 INFO:tasks.rados.rados.0.smithi110.stdout:2708: finishing write tid 6 to smithi11075562-5 2024-09-14T03:46:59.187 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1865 (ObjNum 840 snap 275 seq_num 840) dirty exists 2024-09-14T03:46:59.187 INFO:tasks.rados.rados.0.smithi110.stdout:2708: left oid 5 (ObjNum 840 snap 275 seq_num 840) 2024-09-14T03:46:59.187 INFO:tasks.rados.rados.0.smithi110.stdout:2710: expect (ObjNum 826 snap 274 seq_num 826) 2024-09-14T03:46:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:58 smithi195 ceph-mon[82388]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T03:46:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:46:58 smithi195 ceph-mon[82388]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T03:46:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:58 smithi110 ceph-mon[86423]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T03:46:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:46:58 smithi110 ceph-mon[86423]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T03:46:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:58 smithi110 ceph-mon[88262]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T03:46:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:46:58 smithi110 ceph-mon[88262]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T03:46:59.630 INFO:tasks.rados.rados.0.smithi110.stdout:2712: finishing write tid 1 to smithi11075562-33 2024-09-14T03:46:59.631 INFO:tasks.rados.rados.0.smithi110.stdout:2712: finishing write tid 2 to smithi11075562-33 2024-09-14T03:46:59.631 INFO:tasks.rados.rados.0.smithi110.stdout:2712: finishing write tid 3 to smithi11075562-33 2024-09-14T03:46:59.631 INFO:tasks.rados.rados.0.smithi110.stdout:2712: finishing write tid 4 to smithi11075562-33 2024-09-14T03:46:59.631 INFO:tasks.rados.rados.0.smithi110.stdout:2714: seq_num 843 ranges {1507328=524288,2031616=761856,2793472=671744,3465216=172032} 2024-09-14T03:46:59.641 INFO:tasks.rados.rados.0.smithi110.stdout:2714: writing smithi11075562-46 from 1507328 to 2031616 tid 1 2024-09-14T03:46:59.646 INFO:tasks.rados.rados.0.smithi110.stdout:2714: writing smithi11075562-46 from 2031616 to 2793472 tid 2 2024-09-14T03:46:59.650 INFO:tasks.rados.rados.0.smithi110.stdout:2714: writing smithi11075562-46 from 2793472 to 3465216 tid 3 2024-09-14T03:46:59.653 INFO:tasks.rados.rados.0.smithi110.stdout:2714: writing smithi11075562-46 from 3465216 to 3637248 tid 4 2024-09-14T03:46:59.653 INFO:tasks.rados.rados.0.smithi110.stdout:2705: done (8 left) 2024-09-14T03:46:59.654 INFO:tasks.rados.rados.0.smithi110.stdout:2706: done (7 left) 2024-09-14T03:46:59.654 INFO:tasks.rados.rados.0.smithi110.stdout:2708: done (6 left) 2024-09-14T03:46:59.654 INFO:tasks.rados.rados.0.smithi110.stdout:2709: done (5 left) 2024-09-14T03:46:59.654 INFO:tasks.rados.rados.0.smithi110.stdout:2710: done (4 left) 2024-09-14T03:46:59.654 INFO:tasks.rados.rados.0.smithi110.stdout:2711: done (3 left) 2024-09-14T03:46:59.654 INFO:tasks.rados.rados.0.smithi110.stdout:2715: rollback oid 50 current snap is 275 2024-09-14T03:46:59.654 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 270 2024-09-14T03:46:59.655 INFO:tasks.rados.rados.0.smithi110.stdout:2712: finishing write tid 5 to smithi11075562-33 2024-09-14T03:46:59.655 INFO:tasks.rados.rados.0.smithi110.stdout:2712: finishing write tid 6 to smithi11075562-33 2024-09-14T03:46:59.655 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1602 (ObjNum 841 snap 275 seq_num 841) dirty exists 2024-09-14T03:46:59.655 INFO:tasks.rados.rados.0.smithi110.stdout:2712: left oid 33 (ObjNum 841 snap 275 seq_num 841) 2024-09-14T03:46:59.655 INFO:tasks.rados.rados.0.smithi110.stdout:2713: finishing write tid 1 to smithi11075562-42 2024-09-14T03:46:59.655 INFO:tasks.rados.rados.0.smithi110.stdout:2713: finishing write tid 2 to smithi11075562-42 2024-09-14T03:46:59.656 INFO:tasks.rados.rados.0.smithi110.stdout:2713: finishing write tid 3 to smithi11075562-42 2024-09-14T03:46:59.656 INFO:tasks.rados.rados.0.smithi110.stdout:2712: done (3 left) 2024-09-14T03:46:59.656 INFO:tasks.rados.rados.0.smithi110.stdout:2716: delete oid 43 current snap is 275 2024-09-14T03:46:59.656 INFO:tasks.rados.rados.0.smithi110.stdout:2713: finishing write tid 4 to smithi11075562-42 2024-09-14T03:46:59.656 INFO:tasks.rados.rados.0.smithi110.stdout:2713: finishing write tid 5 to smithi11075562-42 2024-09-14T03:46:59.656 INFO:tasks.rados.rados.0.smithi110.stdout:2713: finishing write tid 6 to smithi11075562-42 2024-09-14T03:46:59.657 INFO:tasks.rados.rados.0.smithi110.stdout:2713: finishing write tid 7 to smithi11075562-42 2024-09-14T03:46:59.657 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1819 (ObjNum 842 snap 275 seq_num 842) dirty exists 2024-09-14T03:46:59.657 INFO:tasks.rados.rados.0.smithi110.stdout:2713: left oid 42 (ObjNum 842 snap 275 seq_num 842) 2024-09-14T03:46:59.657 INFO:tasks.rados.rados.0.smithi110.stdout:2714: finishing write tid 1 to smithi11075562-46 2024-09-14T03:46:59.657 INFO:tasks.rados.rados.0.smithi110.stdout:2714: finishing write tid 2 to smithi11075562-46 2024-09-14T03:46:59.659 INFO:tasks.rados.rados.0.smithi110.stdout:2714: finishing write tid 3 to smithi11075562-46 2024-09-14T03:46:59.659 INFO:tasks.rados.rados.0.smithi110.stdout:2713: done (3 left) 2024-09-14T03:46:59.659 INFO:tasks.rados.rados.0.smithi110.stdout:2716: done (2 left) 2024-09-14T03:46:59.659 INFO:tasks.rados.rados.0.smithi110.stdout:2717: snap_create 2024-09-14T03:46:59.660 INFO:tasks.rados.rados.0.smithi110.stdout:2714: finishing write tid 4 to smithi11075562-46 2024-09-14T03:46:59.660 INFO:tasks.rados.rados.0.smithi110.stdout:2715: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:46:59.660 INFO:tasks.rados.rados.0.smithi110.stdout:2715: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:46:59.660 INFO:tasks.rados.rados.0.smithi110.stdout:2714: finishing write tid 5 to smithi11075562-46 2024-09-14T03:46:59.660 INFO:tasks.rados.rados.0.smithi110.stdout:2715: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:46:59.660 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 1497 (ObjNum 784 snap 257 seq_num 784) dirty exists 2024-09-14T03:46:59.661 INFO:tasks.rados.rados.0.smithi110.stdout:2714: finishing write tid 6 to smithi11075562-46 2024-09-14T03:46:59.661 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1607 (ObjNum 843 snap 275 seq_num 843) dirty exists 2024-09-14T03:46:59.661 INFO:tasks.rados.rados.0.smithi110.stdout:2714: left oid 46 (ObjNum 843 snap 275 seq_num 843) 2024-09-14T03:47:00.543 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:47:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:47:00.557 INFO:tasks.rados.rados.0.smithi110.stdout:2714: done (2 left) 2024-09-14T03:47:00.557 INFO:tasks.rados.rados.0.smithi110.stdout:2715: done (1 left) 2024-09-14T03:47:00.557 INFO:tasks.rados.rados.0.smithi110.stdout:2717: done (0 left) 2024-09-14T03:47:00.557 INFO:tasks.rados.rados.0.smithi110.stdout:2718: copy_from oid 8 from oid 3 current snap is 276 2024-09-14T03:47:00.557 INFO:tasks.rados.rados.0.smithi110.stdout:2719: setattr oid 13 current snap is 276 2024-09-14T03:47:00.559 INFO:tasks.rados.rados.0.smithi110.stdout:2720: delete oid 23 current snap is 276 2024-09-14T03:47:00.560 INFO:tasks.rados.rados.0.smithi110.stdout:2718: finishing copy_from to smithi11075562-8 2024-09-14T03:47:00.560 INFO:tasks.rados.rados.0.smithi110.stdout:2718: got expected ENOENT (src dne) 2024-09-14T03:47:00.560 INFO:tasks.rados.rados.0.smithi110.stdout:2718: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:47:00.562 INFO:tasks.rados.rados.0.smithi110.stdout:2718: done (2 left) 2024-09-14T03:47:00.562 INFO:tasks.rados.rados.0.smithi110.stdout:2720: done (1 left) 2024-09-14T03:47:00.562 INFO:tasks.rados.rados.0.smithi110.stdout:2721: copy_from oid 30 from oid 35 current snap is 276 2024-09-14T03:47:00.562 INFO:tasks.rados.rados.0.smithi110.stdout:2722: snap_create 2024-09-14T03:47:00.563 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1820 (ObjNum -1377854240 snap 32633 seq_num 304) dirty exists 2024-09-14T03:47:00.595 INFO:tasks.rados.rados.0.smithi110.stdout:2721: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:47:00.599 INFO:tasks.rados.rados.0.smithi110.stdout:2721: finishing copy_from to smithi11075562-30 2024-09-14T03:47:00.599 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1808 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:47:00.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-14T03:47:00.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:47:00.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: Upgrade: Finalizing container_image settings 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:47:00.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: Upgrade: Finalizing container_image settings 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:47:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:47:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:47:00.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:47:00.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:47:00.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:47:00.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:47:00.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:47:00.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:47:00.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: Upgrade: Complete! 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:47:00.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: Upgrade: Complete! 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:47:00.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: pgmap v556: 121 pgs: 121 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.9 MiB/s wr, 26 op/s 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: pgmap v556: 121 pgs: 121 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.9 MiB/s wr, 26 op/s 2024-09-14T03:47:00.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.963 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.964 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: Upgrade: Finalizing container_image settings 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:47:00.965 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:47:00.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:47:00.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:47:00.968 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: Upgrade: Complete! 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: pgmap v556: 121 pgs: 121 active+clean; 255 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.9 MiB/s wr, 26 op/s 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:47:00.969 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:01.563 INFO:tasks.rados.rados.0.smithi110.stdout:2719: done (2 left) 2024-09-14T03:47:01.563 INFO:tasks.rados.rados.0.smithi110.stdout:2721: done (1 left) 2024-09-14T03:47:01.564 INFO:tasks.rados.rados.0.smithi110.stdout:2722: done (0 left) 2024-09-14T03:47:01.564 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 277 2024-09-14T03:47:01.564 INFO:tasks.rados.rados.0.smithi110.stdout:2723: seq_num 846 ranges {7135232=573440,7708672=466944,8175616=491520,8667136=548864,9216000=98304} 2024-09-14T03:47:01.603 INFO:tasks.rados.rados.0.smithi110.stdout:2723: writing smithi11075562-12 from 7135232 to 7708672 tid 1 2024-09-14T03:47:01.605 INFO:tasks.rados.rados.0.smithi110.stdout:2723: writing smithi11075562-12 from 7708672 to 8175616 tid 2 2024-09-14T03:47:01.608 INFO:tasks.rados.rados.0.smithi110.stdout:2723: writing smithi11075562-12 from 8175616 to 8667136 tid 3 2024-09-14T03:47:01.611 INFO:tasks.rados.rados.0.smithi110.stdout:2723: writing smithi11075562-12 from 8667136 to 9216000 tid 4 2024-09-14T03:47:01.612 INFO:tasks.rados.rados.0.smithi110.stdout:2723: writing smithi11075562-12 from 9216000 to 9314304 tid 5 2024-09-14T03:47:01.612 INFO:tasks.rados.rados.0.smithi110.stdout:2724: read oid 33 snap -1 2024-09-14T03:47:01.612 INFO:tasks.rados.rados.0.smithi110.stdout:2724: expect (ObjNum 841 snap 275 seq_num 841) 2024-09-14T03:47:01.612 INFO:tasks.rados.rados.0.smithi110.stdout:2725: delete oid 48 current snap is 277 2024-09-14T03:47:01.613 INFO:tasks.rados.rados.0.smithi110.stdout:2723: finishing write tid 1 to smithi11075562-12 2024-09-14T03:47:01.616 INFO:tasks.rados.rados.0.smithi110.stdout:2723: finishing write tid 2 to smithi11075562-12 2024-09-14T03:47:01.617 INFO:tasks.rados.rados.0.smithi110.stdout:2723: finishing write tid 3 to smithi11075562-12 2024-09-14T03:47:01.620 INFO:tasks.rados.rados.0.smithi110.stdout:2725: done (2 left) 2024-09-14T03:47:01.620 INFO:tasks.rados.rados.0.smithi110.stdout:2726: snap_create 2024-09-14T03:47:01.621 INFO:tasks.rados.rados.0.smithi110.stdout:2723: finishing write tid 4 to smithi11075562-12 2024-09-14T03:47:01.623 INFO:tasks.rados.rados.0.smithi110.stdout:2723: finishing write tid 5 to smithi11075562-12 2024-09-14T03:47:01.623 INFO:tasks.rados.rados.0.smithi110.stdout:2723: finishing write tid 6 to smithi11075562-12 2024-09-14T03:47:01.624 INFO:tasks.rados.rados.0.smithi110.stdout:2723: finishing write tid 7 to smithi11075562-12 2024-09-14T03:47:01.625 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1872 (ObjNum 846 snap 277 seq_num 846) dirty exists 2024-09-14T03:47:01.625 INFO:tasks.rados.rados.0.smithi110.stdout:2723: left oid 12 (ObjNum 846 snap 277 seq_num 846) 2024-09-14T03:47:01.648 INFO:tasks.rados.rados.0.smithi110.stdout:2724: expect (ObjNum 841 snap 275 seq_num 841) 2024-09-14T03:47:01.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:01 smithi110 ceph-mon[88262]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T03:47:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:01 smithi110 ceph-mon[86423]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T03:47:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:47:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:47:01] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-14T03:47:01.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:01 smithi195 ceph-mon[82388]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T03:47:02.567 INFO:tasks.rados.rados.0.smithi110.stdout:2723: done (2 left) 2024-09-14T03:47:02.567 INFO:tasks.rados.rados.0.smithi110.stdout:2724: done (1 left) 2024-09-14T03:47:02.567 INFO:tasks.rados.rados.0.smithi110.stdout:2726: done (0 left) 2024-09-14T03:47:02.567 INFO:tasks.rados.rados.0.smithi110.stdout:2727: delete oid 33 current snap is 278 2024-09-14T03:47:02.574 INFO:tasks.rados.rados.0.smithi110.stdout:2727: done (0 left) 2024-09-14T03:47:02.574 INFO:tasks.rados.rados.0.smithi110.stdout:2728: read oid 30 snap -1 2024-09-14T03:47:02.574 INFO:tasks.rados.rados.0.smithi110.stdout:2728: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:47:02.574 INFO:tasks.rados.rados.0.smithi110.stdout:2729: copy_from oid 22 from oid 27 current snap is 278 2024-09-14T03:47:02.574 INFO:tasks.rados.rados.0.smithi110.stdout:2730: read oid 1 snap -1 2024-09-14T03:47:02.574 INFO:tasks.rados.rados.0.smithi110.stdout:2730: expect (ObjNum 820 snap 272 seq_num 820) 2024-09-14T03:47:02.574 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 278 2024-09-14T03:47:02.574 INFO:tasks.rados.rados.0.smithi110.stdout:2731: seq_num 849 ranges {1982464=622592,2605056=516096,3121152=663552,3784704=425984} 2024-09-14T03:47:02.588 INFO:tasks.rados.rados.0.smithi110.stdout:2731: writing smithi11075562-11 from 1982464 to 2605056 tid 1 2024-09-14T03:47:02.591 INFO:tasks.rados.rados.0.smithi110.stdout:2731: writing smithi11075562-11 from 2605056 to 3121152 tid 2 2024-09-14T03:47:02.595 INFO:tasks.rados.rados.0.smithi110.stdout:2731: writing smithi11075562-11 from 3121152 to 3784704 tid 3 2024-09-14T03:47:02.598 INFO:tasks.rados.rados.0.smithi110.stdout:2731: writing smithi11075562-11 from 3784704 to 4210688 tid 4 2024-09-14T03:47:02.598 INFO:tasks.rados.rados.0.smithi110.stdout:2732: read oid 31 snap -1 2024-09-14T03:47:02.598 INFO:tasks.rados.rados.0.smithi110.stdout:2732: expect deleted 2024-09-14T03:47:02.599 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 278 2024-09-14T03:47:02.599 INFO:tasks.rados.rados.0.smithi110.stdout:2729: finishing copy_from to smithi11075562-22 2024-09-14T03:47:02.599 INFO:tasks.rados.rados.0.smithi110.stdout:2729: got expected ENOENT (src dne) 2024-09-14T03:47:02.599 INFO:tasks.rados.rados.0.smithi110.stdout:2729: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:47:02.599 INFO:tasks.rados.rados.0.smithi110.stdout:2731: finishing write tid 1 to smithi11075562-11 2024-09-14T03:47:02.599 INFO:tasks.rados.rados.0.smithi110.stdout:2733: seq_num 850 ranges {0=638976,638976=802816,1441792=475136,1916928=327680} 2024-09-14T03:47:02.602 INFO:tasks.rados.rados.0.smithi110.stdout:2733: writing smithi11075562-36 from 0 to 638976 tid 1 2024-09-14T03:47:02.606 INFO:tasks.rados.rados.0.smithi110.stdout:2733: writing smithi11075562-36 from 638976 to 1441792 tid 2 2024-09-14T03:47:02.609 INFO:tasks.rados.rados.0.smithi110.stdout:2733: writing smithi11075562-36 from 1441792 to 1916928 tid 3 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2733: writing smithi11075562-36 from 1916928 to 2244608 tid 4 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2729: done (5 left) 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2734: copy_from oid 18 from oid 21 current snap is 278 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2731: finishing write tid 2 to smithi11075562-11 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2731: finishing write tid 3 to smithi11075562-11 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2731: finishing write tid 4 to smithi11075562-11 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2732: done (5 left) 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2735: read oid 6 snap -1 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2735: expect (ObjNum 828 snap 274 seq_num 828) 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 278 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2731: finishing write tid 5 to smithi11075562-11 2024-09-14T03:47:02.612 INFO:tasks.rados.rados.0.smithi110.stdout:2731: finishing write tid 6 to smithi11075562-11 2024-09-14T03:47:02.613 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1584 (ObjNum 849 snap 278 seq_num 849) dirty exists 2024-09-14T03:47:02.613 INFO:tasks.rados.rados.0.smithi110.stdout:2731: left oid 11 (ObjNum 849 snap 278 seq_num 849) 2024-09-14T03:47:02.613 INFO:tasks.rados.rados.0.smithi110.stdout:2733: finishing write tid 1 to smithi11075562-36 2024-09-14T03:47:02.613 INFO:tasks.rados.rados.0.smithi110.stdout:2736: seq_num 851 ranges {2506752=655360,3162112=688128,3850240=679936,4530176=237568} 2024-09-14T03:47:02.629 INFO:tasks.rados.rados.0.smithi110.stdout:2736: writing smithi11075562-22 from 2506752 to 3162112 tid 1 2024-09-14T03:47:02.633 INFO:tasks.rados.rados.0.smithi110.stdout:2736: writing smithi11075562-22 from 3162112 to 3850240 tid 2 2024-09-14T03:47:02.637 INFO:tasks.rados.rados.0.smithi110.stdout:2736: writing smithi11075562-22 from 3850240 to 4530176 tid 3 2024-09-14T03:47:02.639 INFO:tasks.rados.rados.0.smithi110.stdout:2736: writing smithi11075562-22 from 4530176 to 4767744 tid 4 2024-09-14T03:47:02.639 INFO:tasks.rados.rados.0.smithi110.stdout:2731: done (6 left) 2024-09-14T03:47:02.639 INFO:tasks.rados.rados.0.smithi110.stdout:2737: snap_remove snap 274 2024-09-14T03:47:02.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:02 smithi110 ceph-mon[88262]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T03:47:02.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:02 smithi110 ceph-mon[88262]: pgmap v559: 121 pgs: 121 active+clean; 269 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 8.9 MiB/s wr, 34 op/s 2024-09-14T03:47:02.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:02 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:02.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:02 smithi110 ceph-mon[86423]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T03:47:02.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:02 smithi110 ceph-mon[86423]: pgmap v559: 121 pgs: 121 active+clean; 269 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 8.9 MiB/s wr, 34 op/s 2024-09-14T03:47:02.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:02 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:02.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:02 smithi195 ceph-mon[82388]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T03:47:02.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:02 smithi195 ceph-mon[82388]: pgmap v559: 121 pgs: 121 active+clean; 269 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 8.9 MiB/s wr, 34 op/s 2024-09-14T03:47:02.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:02 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:47:03.573 INFO:tasks.rados.rados.0.smithi110.stdout:2728: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:47:03.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:03 smithi110 ceph-mon[88262]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T03:47:03.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:03 smithi110 ceph-mon[86423]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T03:47:03.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:03 smithi195 ceph-mon[82388]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T03:47:04.135 INFO:tasks.rados.rados.0.smithi110.stdout:2733: finishing write tid 2 to smithi11075562-36 2024-09-14T03:47:04.135 INFO:tasks.rados.rados.0.smithi110.stdout:2733: finishing write tid 3 to smithi11075562-36 2024-09-14T03:47:04.135 INFO:tasks.rados.rados.0.smithi110.stdout:2733: finishing write tid 4 to smithi11075562-36 2024-09-14T03:47:04.135 INFO:tasks.rados.rados.0.smithi110.stdout:2728: done (6 left) 2024-09-14T03:47:04.135 INFO:tasks.rados.rados.0.smithi110.stdout:2737: done (5 left) 2024-09-14T03:47:04.135 INFO:tasks.rados.rados.0.smithi110.stdout:2738: snap_remove snap 267 2024-09-14T03:47:04.765 INFO:tasks.rados.rados.0.smithi110.stdout:2738: done (5 left) 2024-09-14T03:47:04.765 INFO:tasks.rados.rados.0.smithi110.stdout:2739: delete oid 23 current snap is 278 2024-09-14T03:47:04.766 INFO:tasks.rados.rados.0.smithi110.stdout:2733: finishing write tid 5 to smithi11075562-36 2024-09-14T03:47:04.766 INFO:tasks.rados.rados.0.smithi110.stdout:2733: finishing write tid 6 to smithi11075562-36 2024-09-14T03:47:04.766 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1877 (ObjNum 850 snap 278 seq_num 850) dirty exists 2024-09-14T03:47:04.766 INFO:tasks.rados.rados.0.smithi110.stdout:2733: left oid 36 (ObjNum 850 snap 278 seq_num 850) 2024-09-14T03:47:04.766 INFO:tasks.rados.rados.0.smithi110.stdout:2730: expect (ObjNum 820 snap 272 seq_num 820) 2024-09-14T03:47:04.945 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:04 smithi110 ceph-mon[86423]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T03:47:04.945 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:04 smithi110 ceph-mon[86423]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T03:47:04.945 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:04 smithi110 ceph-mon[86423]: pgmap v563: 121 pgs: 121 active+clean; 275 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 5.8 MiB/s wr, 32 op/s 2024-09-14T03:47:04.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:04 smithi110 ceph-mon[88262]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T03:47:04.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:04 smithi110 ceph-mon[88262]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T03:47:04.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:04 smithi110 ceph-mon[88262]: pgmap v563: 121 pgs: 121 active+clean; 275 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 5.8 MiB/s wr, 32 op/s 2024-09-14T03:47:04.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:04 smithi195 ceph-mon[82388]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T03:47:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:04 smithi195 ceph-mon[82388]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T03:47:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:04 smithi195 ceph-mon[82388]: pgmap v563: 121 pgs: 121 active+clean; 275 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 5.8 MiB/s wr, 32 op/s 2024-09-14T03:47:05.320 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch ps' 2024-09-14T03:47:05.378 INFO:tasks.rados.rados.0.smithi110.stdout:2734: finishing copy_from to smithi11075562-18 2024-09-14T03:47:05.378 INFO:tasks.rados.rados.0.smithi110.stdout:2734: got expected ENOENT (src dne) 2024-09-14T03:47:05.378 INFO:tasks.rados.rados.0.smithi110.stdout:2734: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:47:05.379 INFO:tasks.rados.rados.0.smithi110.stdout:2736: finishing write tid 1 to smithi11075562-22 2024-09-14T03:47:05.379 INFO:tasks.rados.rados.0.smithi110.stdout:2736: finishing write tid 2 to smithi11075562-22 2024-09-14T03:47:05.379 INFO:tasks.rados.rados.0.smithi110.stdout:2735: expect (ObjNum 828 snap 274 seq_num 828) 2024-09-14T03:47:05.488 INFO:tasks.rados.rados.0.smithi110.stdout:2736: finishing write tid 3 to smithi11075562-22 2024-09-14T03:47:05.488 INFO:tasks.rados.rados.0.smithi110.stdout:2736: finishing write tid 4 to smithi11075562-22 2024-09-14T03:47:05.488 INFO:tasks.rados.rados.0.smithi110.stdout:2736: finishing write tid 5 to smithi11075562-22 2024-09-14T03:47:05.488 INFO:tasks.rados.rados.0.smithi110.stdout:2736: finishing write tid 6 to smithi11075562-22 2024-09-14T03:47:05.488 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1813 (ObjNum 851 snap 278 seq_num 851) dirty exists 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2736: left oid 22 (ObjNum 851 snap 278 seq_num 851) 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2730: done (5 left) 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2733: done (4 left) 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2734: done (3 left) 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2735: done (2 left) 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2736: done (1 left) 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2739: done (0 left) 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2740: rollback oid 41 current snap is 278 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 252 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2741: copy_from oid 2 from oid 3 current snap is 278 2024-09-14T03:47:05.489 INFO:tasks.rados.rados.0.smithi110.stdout:2742: delete oid 12 current snap is 278 2024-09-14T03:47:05.490 INFO:tasks.rados.rados.0.smithi110.stdout:2741: finishing copy_from to smithi11075562-2 2024-09-14T03:47:05.490 INFO:tasks.rados.rados.0.smithi110.stdout:2741: got expected ENOENT (src dne) 2024-09-14T03:47:05.490 INFO:tasks.rados.rados.0.smithi110.stdout:2741: finishing copy_from racing read to smithi11075562-2 2024-09-14T03:47:05.491 INFO:tasks.rados.rados.0.smithi110.stdout:2740: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:47:05.491 INFO:tasks.rados.rados.0.smithi110.stdout:2741: done (2 left) 2024-09-14T03:47:05.491 INFO:tasks.rados.rados.0.smithi110.stdout:2742: done (1 left) 2024-09-14T03:47:05.492 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 278 2024-09-14T03:47:05.492 INFO:tasks.rados.rados.0.smithi110.stdout:2743: seq_num 854 ranges {1572864=712704,2285568=794624,3080192=450560,3530752=360448} 2024-09-14T03:47:05.504 INFO:tasks.rados.rados.0.smithi110.stdout:2743: writing smithi11075562-24 from 1572864 to 2285568 tid 1 2024-09-14T03:47:05.508 INFO:tasks.rados.rados.0.smithi110.stdout:2743: writing smithi11075562-24 from 2285568 to 3080192 tid 2 2024-09-14T03:47:05.511 INFO:tasks.rados.rados.0.smithi110.stdout:2743: writing smithi11075562-24 from 3080192 to 3530752 tid 3 2024-09-14T03:47:05.513 INFO:tasks.rados.rados.0.smithi110.stdout:2743: writing smithi11075562-24 from 3530752 to 3891200 tid 4 2024-09-14T03:47:05.513 INFO:tasks.rados.rados.0.smithi110.stdout:2744: copy_from oid 10 from oid 36 current snap is 278 2024-09-14T03:47:05.513 INFO:tasks.rados.rados.0.smithi110.stdout:2740: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:47:05.513 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1600 (ObjNum 865 snap 0 seq_num 2917113904) dirty dne 2024-09-14T03:47:05.513 INFO:tasks.rados.rados.0.smithi110.stdout:2740: done (2 left) 2024-09-14T03:47:05.513 INFO:tasks.rados.rados.0.smithi110.stdout:2745: snap_remove snap 266 2024-09-14T03:47:05.611 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:05.771 INFO:tasks.rados.rados.0.smithi110.stdout:2745: done (2 left) 2024-09-14T03:47:05.771 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 278 2024-09-14T03:47:05.771 INFO:tasks.rados.rados.0.smithi110.stdout:2746: seq_num 855 ranges {0=729088,729088=671744,1400832=663552,2064384=270336} 2024-09-14T03:47:05.776 INFO:tasks.rados.rados.0.smithi110.stdout:2746: writing smithi11075562-41 from 0 to 729088 tid 1 2024-09-14T03:47:05.781 INFO:tasks.rados.rados.0.smithi110.stdout:2746: writing smithi11075562-41 from 729088 to 1400832 tid 2 2024-09-14T03:47:05.786 INFO:tasks.rados.rados.0.smithi110.stdout:2746: writing smithi11075562-41 from 1400832 to 2064384 tid 3 2024-09-14T03:47:05.788 INFO:tasks.rados.rados.0.smithi110.stdout:2746: writing smithi11075562-41 from 2064384 to 2334720 tid 4 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2747: delete oid 27 current snap is 278 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2743: finishing write tid 1 to smithi11075562-24 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2743: finishing write tid 2 to smithi11075562-24 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2743: finishing write tid 3 to smithi11075562-24 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2743: finishing write tid 4 to smithi11075562-24 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2743: finishing write tid 5 to smithi11075562-24 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2743: finishing write tid 6 to smithi11075562-24 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1513 (ObjNum 854 snap 278 seq_num 854) dirty exists 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2743: left oid 24 (ObjNum 854 snap 278 seq_num 854) 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2744: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2744: finishing copy_from to smithi11075562-10 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1877 (ObjNum 850 snap 278 seq_num 850) dirty exists 2024-09-14T03:47:05.789 INFO:tasks.rados.rados.0.smithi110.stdout:2746: finishing write tid 1 to smithi11075562-41 2024-09-14T03:47:05.790 INFO:tasks.rados.rados.0.smithi110.stdout:2746: finishing write tid 2 to smithi11075562-41 2024-09-14T03:47:05.791 INFO:tasks.rados.rados.0.smithi110.stdout:2743: done (3 left) 2024-09-14T03:47:05.791 INFO:tasks.rados.rados.0.smithi110.stdout:2744: done (2 left) 2024-09-14T03:47:05.791 INFO:tasks.rados.rados.0.smithi110.stdout:2747: done (1 left) 2024-09-14T03:47:05.792 INFO:tasks.rados.rados.0.smithi110.stdout:2748: delete oid 28 current snap is 278 2024-09-14T03:47:05.793 INFO:tasks.rados.rados.0.smithi110.stdout:2746: finishing write tid 3 to smithi11075562-41 2024-09-14T03:47:05.794 INFO:tasks.rados.rados.0.smithi110.stdout:2746: finishing write tid 4 to smithi11075562-41 2024-09-14T03:47:05.794 INFO:tasks.rados.rados.0.smithi110.stdout:2748: done (1 left) 2024-09-14T03:47:05.794 INFO:tasks.rados.rados.0.smithi110.stdout:2749: snap_remove snap 264 2024-09-14T03:47:06.008 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:05 smithi110 ceph-mon[88262]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:06.008 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:05 smithi110 ceph-mon[88262]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T03:47:06.008 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:05 smithi110 ceph-mon[86423]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:06.008 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:05 smithi110 ceph-mon[86423]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T03:47:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:05 smithi195 ceph-mon[82388]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:05 smithi195 ceph-mon[82388]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T03:47:06.782 INFO:tasks.rados.rados.0.smithi110.stdout:2749: done (1 left) 2024-09-14T03:47:06.782 INFO:tasks.rados.rados.0.smithi110.stdout:2750: snap_create 2024-09-14T03:47:06.782 INFO:tasks.rados.rados.0.smithi110.stdout:2746: finishing write tid 5 to smithi11075562-41 2024-09-14T03:47:06.782 INFO:tasks.rados.rados.0.smithi110.stdout:2746: finishing write tid 6 to smithi11075562-41 2024-09-14T03:47:06.782 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1605 (ObjNum 855 snap 278 seq_num 855) dirty exists 2024-09-14T03:47:06.782 INFO:tasks.rados.rados.0.smithi110.stdout:2746: left oid 41 (ObjNum 855 snap 278 seq_num 855) 2024-09-14T03:47:07.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:06 smithi110 ceph-mon[86423]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T03:47:07.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:06 smithi110 ceph-mon[86423]: pgmap v566: 121 pgs: 121 active+clean; 291 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 6.3 MiB/s wr, 21 op/s 2024-09-14T03:47:07.116 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:06 smithi110 ceph-mon[88262]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T03:47:07.116 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:06 smithi110 ceph-mon[88262]: pgmap v566: 121 pgs: 121 active+clean; 291 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 6.3 MiB/s wr, 21 op/s 2024-09-14T03:47:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:06 smithi195 ceph-mon[82388]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T03:47:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:06 smithi195 ceph-mon[82388]: pgmap v566: 121 pgs: 121 active+clean; 291 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 6.3 MiB/s wr, 21 op/s 2024-09-14T03:47:07.301 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:47:07.301 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (7m) 57s ago 12m 19.7M - 0.27.0 11f11916f8cd 02afc2b875a4 2024-09-14T03:47:07.301 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi195 *:3000 running (18s) 11s ago 11m 67.1M - 10.4.0 c8b91775d855 c9fbdb64fe96 2024-09-14T03:47:07.301 INFO:teuthology.orchestra.run.smithi110.stdout:mds.a smithi110 running (61s) 57s ago 13m 13.2M - 19.3.0-4907-ga2782315 8801d2529e43 ddd26c7ac6c2 2024-09-14T03:47:07.301 INFO:teuthology.orchestra.run.smithi110.stdout:mds.b smithi195 running (39s) 11s ago 13m 15.5M - 19.3.0-4907-ga2782315 8801d2529e43 08047cadae86 2024-09-14T03:47:07.301 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi195 *:8443,9283,8765 running (6m) 11s ago 16m 455M - 19.3.0-4907-ga2782315 8801d2529e43 fbeb4897d868 2024-09-14T03:47:07.301 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (7m) 57s ago 18m 530M - 19.3.0-4907-ga2782315 8801d2529e43 f1f945bddfa6 2024-09-14T03:47:07.301 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (6m) 57s ago 18m 68.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 c325cb2844f4 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi195 running (5m) 11s ago 17m 55.4M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4992d17f309c 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (6m) 57s ago 16m 58.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a2cf589f8f7e 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (7m) 57s ago 11m 10.1M - 1.7.0 72c9c2088986 1b0d2ce16092 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi195 *:9100 running (7m) 11s ago 11m 11.9M - 1.7.0 72c9c2088986 e217e492de5d 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (5m) 57s ago 16m 274M 2480M 19.3.0-4907-ga2782315 8801d2529e43 9626562f2155 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (4m) 57s ago 15m 182M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ed2d900d1e2 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (4m) 57s ago 15m 189M 2480M 19.3.0-4907-ga2782315 8801d2529e43 0cdc58c38711 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (3m) 57s ago 15m 139M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2283ebcb5f10 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi195 running (3m) 11s ago 14m 257M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6921b0a30047 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi195 running (2m) 11s ago 14m 208M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8fe7f08d5882 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi195 running (114s) 11s ago 14m 127M 2480M 19.3.0-4907-ga2782315 8801d2529e43 5305ca81bc26 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi195 running (80s) 11s ago 13m 134M 2480M 19.3.0-4907-ga2782315 8801d2529e43 98e6e5252ce4 2024-09-14T03:47:07.302 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi195 *:9095 running (6m) 11s ago 12m 54.9M - 2.51.0 1d3b7f56885b e0fe3bbc1c08 2024-09-14T03:47:07.781 INFO:tasks.rados.rados.0.smithi110.stdout:2746: done (1 left) 2024-09-14T03:47:07.781 INFO:tasks.rados.rados.0.smithi110.stdout:2750: done (0 left) 2024-09-14T03:47:07.781 INFO:tasks.rados.rados.0.smithi110.stdout:2751: rmattr oid 49 current snap is 279 2024-09-14T03:47:07.781 INFO:tasks.rados.rados.0.smithi110.stdout:2751: done (0 left) 2024-09-14T03:47:07.781 INFO:tasks.rados.rados.0.smithi110.stdout:2752: snap_remove snap 268 2024-09-14T03:47:08.036 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:07 smithi110 ceph-mon[88262]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T03:47:08.037 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:07 smithi110 ceph-mon[86423]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T03:47:08.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph versions' 2024-09-14T03:47:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:07 smithi195 ceph-mon[82388]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T03:47:08.352 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:08.763 INFO:tasks.rados.rados.0.smithi110.stdout:2752: done (0 left) 2024-09-14T03:47:08.763 INFO:tasks.rados.rados.0.smithi110.stdout:2753: rollback oid 11 current snap is 279 2024-09-14T03:47:08.764 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 251 2024-09-14T03:47:08.764 INFO:tasks.rados.rados.0.smithi110.stdout:2754: delete oid 36 current snap is 279 2024-09-14T03:47:08.776 INFO:tasks.rados.rados.0.smithi110.stdout:2753: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:47:08.776 INFO:tasks.rados.rados.0.smithi110.stdout:2753: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:47:08.776 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1586 (ObjNum 225 snap 0 seq_num 66685536) dirty dne 2024-09-14T03:47:08.780 INFO:tasks.rados.rados.0.smithi110.stdout:2753: done (1 left) 2024-09-14T03:47:08.780 INFO:tasks.rados.rados.0.smithi110.stdout:2754: done (0 left) 2024-09-14T03:47:08.780 INFO:tasks.rados.rados.0.smithi110.stdout:2755: copy_from oid 45 from oid 47 current snap is 279 2024-09-14T03:47:08.781 INFO:tasks.rados.rados.0.smithi110.stdout:2756: snap_remove snap 278 2024-09-14T03:47:08.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:08 smithi110 ceph-mon[88262]: from='client.44483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:08.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:08 smithi110 ceph-mon[88262]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T03:47:08.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:08 smithi110 ceph-mon[88262]: pgmap v569: 121 pgs: 121 active+clean; 293 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 12 op/s 2024-09-14T03:47:08.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:08 smithi110 ceph-mon[88262]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T03:47:08.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:08 smithi110 ceph-mon[86423]: from='client.44483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:08.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:08 smithi110 ceph-mon[86423]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T03:47:08.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:08 smithi110 ceph-mon[86423]: pgmap v569: 121 pgs: 121 active+clean; 293 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 12 op/s 2024-09-14T03:47:08.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:08 smithi110 ceph-mon[86423]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T03:47:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:08 smithi195 ceph-mon[82388]: from='client.44483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:08 smithi195 ceph-mon[82388]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T03:47:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:08 smithi195 ceph-mon[82388]: pgmap v569: 121 pgs: 121 active+clean; 293 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 12 op/s 2024-09-14T03:47:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:08 smithi195 ceph-mon[82388]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T03:47:09.784 INFO:tasks.rados.rados.0.smithi110.stdout:2756: done (1 left) 2024-09-14T03:47:09.784 INFO:tasks.rados.rados.0.smithi110.stdout:2757: setattr oid 1 current snap is 279 2024-09-14T03:47:09.784 INFO:tasks.rados.rados.0.smithi110.stdout:2755: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:47:09.784 INFO:tasks.rados.rados.0.smithi110.stdout:2755: finishing copy_from to smithi11075562-45 2024-09-14T03:47:09.784 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 1859 (ObjNum 838 snap 275 seq_num 838) dirty exists 2024-09-14T03:47:09.785 INFO:tasks.rados.rados.0.smithi110.stdout:2755: done (1 left) 2024-09-14T03:47:09.785 INFO:tasks.rados.rados.0.smithi110.stdout:2758: snap_create 2024-09-14T03:47:09.790 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1821 (ObjNum 820 snap 272 seq_num 820) dirty exists 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2024-09-14T03:47:10.106 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:47:10.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:47:10.107 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2024-09-14T03:47:10.107 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:47:10.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2024-09-14T03:47:10.107 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2024-09-14T03:47:10.107 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 15 2024-09-14T03:47:10.107 INFO:teuthology.orchestra.run.smithi110.stdout: } 2024-09-14T03:47:10.107 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-14T03:47:10.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:47:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:47:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:47:10.717 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-14T03:47:10.789 INFO:tasks.rados.rados.0.smithi110.stdout:2757: done (1 left) 2024-09-14T03:47:10.789 INFO:tasks.rados.rados.0.smithi110.stdout:2758: done (0 left) 2024-09-14T03:47:10.789 INFO:tasks.rados.rados.0.smithi110.stdout:2759: snap_remove snap 238 2024-09-14T03:47:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:10 smithi110 ceph-mon[86423]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T03:47:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:10 smithi110 ceph-mon[86423]: pgmap v572: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-14T03:47:10.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:10 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2894394583' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:10.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:10 smithi110 ceph-mon[88262]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T03:47:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:10 smithi110 ceph-mon[88262]: pgmap v572: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-14T03:47:10.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:10 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2894394583' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:10.963 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:11.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:10 smithi195 ceph-mon[82388]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T03:47:11.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:10 smithi195 ceph-mon[82388]: pgmap v572: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-14T03:47:11.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:10 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2894394583' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:11.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:47:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:47:11] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-14T03:47:11.791 INFO:tasks.rados.rados.0.smithi110.stdout:2759: done (0 left) 2024-09-14T03:47:11.791 INFO:tasks.rados.rados.0.smithi110.stdout:2760: delete oid 29 current snap is 280 2024-09-14T03:47:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2760: done (0 left) 2024-09-14T03:47:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2761: copy_from oid 14 from oid 22 current snap is 280 2024-09-14T03:47:11.802 INFO:tasks.rados.rados.0.smithi110.stdout:2762: delete oid 40 current snap is 280 2024-09-14T03:47:11.809 INFO:tasks.rados.rados.0.smithi110.stdout:2762: done (1 left) 2024-09-14T03:47:11.809 INFO:tasks.rados.rados.0.smithi110.stdout:2763: delete oid 21 current snap is 280 2024-09-14T03:47:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:2763: done (1 left) 2024-09-14T03:47:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:2764: rmattr oid 28 current snap is 280 2024-09-14T03:47:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:2764: done (1 left) 2024-09-14T03:47:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:2765: read oid 7 snap 276 2024-09-14T03:47:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:2765: expect (ObjNum 816 snap 270 seq_num 816) 2024-09-14T03:47:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:2766: read oid 16 snap -1 2024-09-14T03:47:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:2766: expect deleted 2024-09-14T03:47:11.814 INFO:tasks.rados.rados.0.smithi110.stdout:2767: snap_remove snap 250 2024-09-14T03:47:12.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:11 smithi110 ceph-mon[86423]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T03:47:12.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:11 smithi110 ceph-mon[88262]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T03:47:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:11 smithi195 ceph-mon[82388]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T03:47:12.726 INFO:teuthology.orchestra.run.smithi110.stdout:true 2024-09-14T03:47:12.803 INFO:tasks.rados.rados.0.smithi110.stdout:2767: done (3 left) 2024-09-14T03:47:12.803 INFO:tasks.rados.rados.0.smithi110.stdout:2768: snap_create 2024-09-14T03:47:12.803 INFO:tasks.rados.rados.0.smithi110.stdout:2765: expect (ObjNum 816 snap 270 seq_num 816) 2024-09-14T03:47:12.871 INFO:tasks.rados.rados.0.smithi110.stdout:2761: finishing copy_from racing read to smithi11075562-14 2024-09-14T03:47:12.872 INFO:tasks.rados.rados.0.smithi110.stdout:2761: finishing copy_from to smithi11075562-14 2024-09-14T03:47:12.872 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1813 (ObjNum 851 snap 278 seq_num 851) dirty exists 2024-09-14T03:47:13.080 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:12 smithi110 ceph-mon[86423]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T03:47:13.080 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:12 smithi110 ceph-mon[86423]: pgmap v575: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-14T03:47:13.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:13.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:12 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1178062120' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:13.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:12 smithi110 ceph-mon[88262]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T03:47:13.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:12 smithi110 ceph-mon[88262]: pgmap v575: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-14T03:47:13.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:13.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:12 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1178062120' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:12 smithi195 ceph-mon[82388]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T03:47:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:12 smithi195 ceph-mon[82388]: pgmap v575: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-14T03:47:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:12 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1178062120' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:13.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5517b266-7249-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-14T03:47:13.584 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:13.766 INFO:tasks.rados.rados.0.smithi110.stdout:2761: done (3 left) 2024-09-14T03:47:13.766 INFO:tasks.rados.rados.0.smithi110.stdout:2765: done (2 left) 2024-09-14T03:47:13.767 INFO:tasks.rados.rados.0.smithi110.stdout:2766: done (1 left) 2024-09-14T03:47:13.767 INFO:tasks.rados.rados.0.smithi110.stdout:2768: done (0 left) 2024-09-14T03:47:13.767 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 281 2024-09-14T03:47:13.767 INFO:tasks.rados.rados.0.smithi110.stdout:2769: seq_num 862 ranges {4767744=40960} 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:2769: writing smithi11075562-22 from 4767744 to 4808704 tid 1 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:2770: read oid 40 snap -1 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:2770: expect deleted 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:2771: rollback oid 16 current snap is 281 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 270 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:2772: read oid 15 snap -1 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:2772: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 281 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:2773: seq_num 863 ranges {0=57344} 2024-09-14T03:47:13.794 INFO:tasks.rados.rados.0.smithi110.stdout:2773: writing smithi11075562-3 from 0 to 57344 tid 1 2024-09-14T03:47:13.795 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 281 2024-09-14T03:47:13.795 INFO:tasks.rados.rados.0.smithi110.stdout:2774: seq_num 864 ranges {1712128=73728} 2024-09-14T03:47:13.803 INFO:tasks.rados.rados.0.smithi110.stdout:2774: writing smithi11075562-44 from 1712128 to 1785856 tid 1 2024-09-14T03:47:13.803 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 281 2024-09-14T03:47:13.803 INFO:tasks.rados.rados.0.smithi110.stdout:2775: seq_num 865 ranges {0=90112} 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:2775: writing smithi11075562-21 from 0 to 90112 tid 1 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 281 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:2776: seq_num 866 ranges {0=106496} 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:2776: writing smithi11075562-13 from 0 to 106496 tid 1 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:2777: read oid 26 snap -1 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:2777: expect (ObjNum 829 snap 274 seq_num 829) 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:2778: rmattr oid 50 current snap is 281 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:2778: done (9 left) 2024-09-14T03:47:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 281 2024-09-14T03:47:13.806 INFO:tasks.rados.rados.0.smithi110.stdout:2779: seq_num 867 ranges {8388608=122880} 2024-09-14T03:47:13.853 INFO:tasks.rados.rados.0.smithi110.stdout:2779: writing smithi11075562-42 from 8388608 to 8511488 tid 1 2024-09-14T03:47:13.853 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 281 2024-09-14T03:47:13.853 INFO:tasks.rados.rados.0.smithi110.stdout:2771: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:47:13.853 INFO:tasks.rados.rados.0.smithi110.stdout:2771: finishing rollback tid 2 to smithi11075562-16 2024-09-14T03:47:13.853 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 1515 (ObjNum 225 snap 0 seq_num 64635392) dirty exists 2024-09-14T03:47:13.853 INFO:tasks.rados.rados.0.smithi110.stdout:2772: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:13.853 INFO:tasks.rados.rados.0.smithi110.stdout:2769: finishing write tid 1 to smithi11075562-22 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2773: finishing write tid 1 to smithi11075562-3 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2773: finishing write tid 2 to smithi11075562-3 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2769: finishing write tid 2 to smithi11075562-22 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2769: finishing write tid 3 to smithi11075562-22 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1815 (ObjNum 862 snap 281 seq_num 862) dirty exists 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2769: left oid 22 (ObjNum 862 snap 281 seq_num 862) 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2774: finishing write tid 1 to smithi11075562-44 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2774: finishing write tid 2 to smithi11075562-44 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2774: finishing write tid 3 to smithi11075562-44 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 420 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2774: left oid 44 (ObjNum 864 snap 281 seq_num 864) 2024-09-14T03:47:13.854 INFO:tasks.rados.rados.0.smithi110.stdout:2780: seq_num 868 ranges {0=139264} 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2780: writing smithi11075562-12 from 0 to 139264 tid 1 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2769: done (10 left) 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2770: done (9 left) 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2771: done (8 left) 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2772: done (7 left) 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2774: done (6 left) 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2781: read oid 17 snap -1 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2781: expect (ObjNum 754 snap 244 seq_num 754) 2024-09-14T03:47:13.855 INFO:tasks.rados.rados.0.smithi110.stdout:2782: snap_remove snap 276 2024-09-14T03:47:14.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:13 smithi110 ceph-mon[88262]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T03:47:14.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:13 smithi110 ceph-mon[88262]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T03:47:14.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:13 smithi110 ceph-mon[86423]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T03:47:14.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:13 smithi110 ceph-mon[86423]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T03:47:14.124 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:13 smithi195 ceph-mon[82388]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T03:47:14.124 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:13 smithi195 ceph-mon[82388]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T03:47:14.812 INFO:tasks.rados.rados.0.smithi110.stdout:2782: done (7 left) 2024-09-14T03:47:14.812 INFO:tasks.rados.rados.0.smithi110.stdout:2783: setattr oid 40 current snap is 281 2024-09-14T03:47:14.812 INFO:tasks.rados.rados.0.smithi110.stdout:2773: finishing write tid 3 to smithi11075562-3 2024-09-14T03:47:14.812 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 1650 (ObjNum 863 snap 281 seq_num 863) dirty exists 2024-09-14T03:47:14.812 INFO:tasks.rados.rados.0.smithi110.stdout:2773: left oid 3 (ObjNum 863 snap 281 seq_num 863) 2024-09-14T03:47:14.812 INFO:tasks.rados.rados.0.smithi110.stdout:2776: finishing write tid 1 to smithi11075562-13 2024-09-14T03:47:14.812 INFO:tasks.rados.rados.0.smithi110.stdout:2776: finishing write tid 2 to smithi11075562-13 2024-09-14T03:47:14.812 INFO:tasks.rados.rados.0.smithi110.stdout:2775: finishing write tid 1 to smithi11075562-21 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2776: finishing write tid 3 to smithi11075562-13 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1823 (ObjNum 866 snap 281 seq_num 866) dirty exists 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2776: left oid 13 (ObjNum 866 snap 281 seq_num 866) 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2775: finishing write tid 2 to smithi11075562-21 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2775: finishing write tid 3 to smithi11075562-21 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 1815 (ObjNum 865 snap 281 seq_num 865) dirty exists 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2775: left oid 21 (ObjNum 865 snap 281 seq_num 865) 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2779: finishing write tid 1 to smithi11075562-42 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2779: finishing write tid 2 to smithi11075562-42 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2779: finishing write tid 3 to smithi11075562-42 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1825 (ObjNum 867 snap 281 seq_num 867) dirty exists 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2779: left oid 42 (ObjNum 867 snap 281 seq_num 867) 2024-09-14T03:47:14.813 INFO:tasks.rados.rados.0.smithi110.stdout:2781: expect (ObjNum 754 snap 244 seq_num 754) 2024-09-14T03:47:14.833 INFO:tasks.rados.rados.0.smithi110.stdout:2777: expect (ObjNum 829 snap 274 seq_num 829) 2024-09-14T03:47:14.937 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:14 smithi110 ceph-mon[88262]: pgmap v578: 121 pgs: 121 active+clean; 296 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-14T03:47:14.938 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:14 smithi110 ceph-mon[86423]: pgmap v578: 121 pgs: 121 active+clean; 296 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-14T03:47:14.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:14 smithi195 ceph-mon[82388]: pgmap v578: 121 pgs: 121 active+clean; 296 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-14T03:47:15.412 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)" 2024-09-14T03:47:16.006 INFO:tasks.rados.rados.0.smithi110.stdout:2780: finishing write tid 1 to smithi11075562-12 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2780: finishing write tid 2 to smithi11075562-12 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2780: finishing write tid 3 to smithi11075562-12 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1940 (ObjNum 868 snap 281 seq_num 868) dirty exists 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2780: left oid 12 (ObjNum 868 snap 281 seq_num 868) 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2773: done (7 left) 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2775: done (6 left) 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2776: done (5 left) 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2777: done (4 left) 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2779: done (3 left) 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2780: done (2 left) 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:2781: done (1 left) 2024-09-14T03:47:16.007 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 281 2024-09-14T03:47:16.008 INFO:tasks.rados.rados.0.smithi110.stdout:2784: seq_num 869 ranges {4317184=155648} 2024-09-14T03:47:16.029 INFO:tasks.rados.rados.0.smithi110.stdout:2784: writing smithi11075562-49 from 4317184 to 4472832 tid 1 2024-09-14T03:47:16.030 INFO:tasks.rados.rados.0.smithi110.stdout:2785: snap_remove snap 244 2024-09-14T03:47:16.048 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:15 smithi110 ceph-mon[88262]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T03:47:16.048 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:15 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1427491743' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:16.048 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:15 smithi110 ceph-mon[88262]: pgmap v580: 121 pgs: 121 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-09-14T03:47:16.048 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:15 smithi110 ceph-mon[86423]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T03:47:16.048 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:15 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1427491743' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:16.048 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:15 smithi110 ceph-mon[86423]: pgmap v580: 121 pgs: 121 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-09-14T03:47:16.071 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-14T03:47:16.071 INFO:teuthology.task.print:**** done end upgrade, wait... 2024-09-14T03:47:16.071 DEBUG:teuthology.parallel:result is None 2024-09-14T03:47:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:15 smithi195 ceph-mon[82388]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T03:47:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:15 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1427491743' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:47:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:15 smithi195 ceph-mon[82388]: pgmap v580: 121 pgs: 121 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1516 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:2784: finishing write tid 1 to smithi11075562-49 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:2784: finishing write tid 2 to smithi11075562-49 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:2784: finishing write tid 3 to smithi11075562-49 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1827 (ObjNum 869 snap 281 seq_num 869) dirty exists 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:2784: left oid 49 (ObjNum 869 snap 281 seq_num 869) 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:2783: done (2 left) 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:2784: done (1 left) 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:2785: done (0 left) 2024-09-14T03:47:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:2786: snap_remove snap 280 2024-09-14T03:47:17.829 INFO:tasks.rados.rados.0.smithi110.stdout:2786: done (0 left) 2024-09-14T03:47:17.829 INFO:tasks.rados.rados.0.smithi110.stdout:2787: delete oid 28 current snap is 281 2024-09-14T03:47:17.832 INFO:tasks.rados.rados.0.smithi110.stdout:2787: done (0 left) 2024-09-14T03:47:17.832 INFO:tasks.rados.rados.0.smithi110.stdout:2788: delete oid 19 current snap is 281 2024-09-14T03:47:17.836 INFO:tasks.rados.rados.0.smithi110.stdout:2788: done (0 left) 2024-09-14T03:47:17.836 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 281 2024-09-14T03:47:17.836 INFO:tasks.rados.rados.0.smithi110.stdout:2789: seq_num 872 ranges {0=212992} 2024-09-14T03:47:17.838 INFO:tasks.rados.rados.0.smithi110.stdout:2789: writing smithi11075562-18 from 0 to 212992 tid 1 2024-09-14T03:47:17.838 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 281 2024-09-14T03:47:17.839 INFO:tasks.rados.rados.0.smithi110.stdout:2790: seq_num 873 ranges {4767744=229376} 2024-09-14T03:47:17.866 INFO:tasks.rados.rados.0.smithi110.stdout:2790: writing smithi11075562-14 from 4767744 to 4997120 tid 1 2024-09-14T03:47:17.866 INFO:tasks.rados.rados.0.smithi110.stdout:2791: delete oid 35 current snap is 281 2024-09-14T03:47:17.866 INFO:tasks.rados.rados.0.smithi110.stdout:2789: finishing write tid 1 to smithi11075562-18 2024-09-14T03:47:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:2789: finishing write tid 2 to smithi11075562-18 2024-09-14T03:47:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:2789: finishing write tid 3 to smithi11075562-18 2024-09-14T03:47:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1519 (ObjNum 872 snap 281 seq_num 872) dirty exists 2024-09-14T03:47:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:2789: left oid 18 (ObjNum 872 snap 281 seq_num 872) 2024-09-14T03:47:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:2790: finishing write tid 1 to smithi11075562-14 2024-09-14T03:47:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:2789: done (2 left) 2024-09-14T03:47:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:2791: done (1 left) 2024-09-14T03:47:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:2792: setattr oid 35 current snap is 281 2024-09-14T03:47:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:2790: finishing write tid 2 to smithi11075562-14 2024-09-14T03:47:17.872 INFO:tasks.rados.rados.0.smithi110.stdout:2793: copy_from oid 26 from oid 1 current snap is 281 2024-09-14T03:47:17.872 INFO:tasks.rados.rados.0.smithi110.stdout:2794: copy_from oid 17 from oid 29 current snap is 281 2024-09-14T03:47:17.872 INFO:tasks.rados.rados.0.smithi110.stdout:2790: finishing write tid 3 to smithi11075562-14 2024-09-14T03:47:17.872 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1817 (ObjNum 873 snap 281 seq_num 873) dirty exists 2024-09-14T03:47:17.872 INFO:tasks.rados.rados.0.smithi110.stdout:2790: left oid 14 (ObjNum 873 snap 281 seq_num 873) 2024-09-14T03:47:17.872 INFO:tasks.rados.rados.0.smithi110.stdout:2790: done (3 left) 2024-09-14T03:47:17.872 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 281 2024-09-14T03:47:17.872 INFO:tasks.rados.rados.0.smithi110.stdout:2795: seq_num 875 ranges {15400960=262144} 2024-09-14T03:47:17.957 INFO:tasks.rados.rados.0.smithi110.stdout:2795: writing smithi11075562-32 from 15400960 to 15663104 tid 1 2024-09-14T03:47:17.958 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 281 2024-09-14T03:47:17.958 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1829 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:47:17.958 INFO:tasks.rados.rados.0.smithi110.stdout:2796: seq_num 876 ranges {0=278528} 2024-09-14T03:47:17.959 INFO:tasks.rados.rados.0.smithi110.stdout:2796: writing smithi11075562-28 from 0 to 278528 tid 1 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2792: done (4 left) 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2797: read oid 4 snap -1 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2797: expect (ObjNum 796 snap 264 seq_num 796) 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2798: rmattr oid 18 current snap is 281 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2794: finishing copy_from to smithi11075562-17 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2794: got expected ENOENT (src dne) 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2794: finishing copy_from racing read to smithi11075562-17 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2793: finishing copy_from racing read to smithi11075562-26 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:2793: finishing copy_from to smithi11075562-26 2024-09-14T03:47:17.960 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1821 (ObjNum 820 snap 272 seq_num 820) dirty exists 2024-09-14T03:47:17.961 INFO:tasks.rados.rados.0.smithi110.stdout:2793: done (5 left) 2024-09-14T03:47:17.961 INFO:tasks.rados.rados.0.smithi110.stdout:2794: done (4 left) 2024-09-14T03:47:17.961 INFO:tasks.rados.rados.0.smithi110.stdout:2798: done (3 left) 2024-09-14T03:47:17.961 INFO:tasks.rados.rados.0.smithi110.stdout:2799: rollback oid 14 current snap is 281 2024-09-14T03:47:17.961 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 14 to 262 2024-09-14T03:47:17.961 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 281 2024-09-14T03:47:17.961 INFO:tasks.rados.rados.0.smithi110.stdout:2800: seq_num 877 ranges {0=294912} 2024-09-14T03:47:17.962 INFO:tasks.rados.rados.0.smithi110.stdout:2800: writing smithi11075562-2 from 0 to 294912 tid 1 2024-09-14T03:47:17.962 INFO:tasks.rados.rados.0.smithi110.stdout:2801: read oid 25 snap -1 2024-09-14T03:47:17.962 INFO:tasks.rados.rados.0.smithi110.stdout:2801: expect deleted 2024-09-14T03:47:17.962 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 281 2024-09-14T03:47:17.962 INFO:tasks.rados.rados.0.smithi110.stdout:2802: seq_num 878 ranges {8511488=311296} 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2802: writing smithi11075562-42 from 8511488 to 8822784 tid 1 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2803: copy_from oid 46 from oid 26 current snap is 281 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2795: finishing write tid 1 to smithi11075562-32 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2795: finishing write tid 2 to smithi11075562-32 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2799: finishing rollback tid 0 to smithi11075562-14 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2799: finishing rollback tid 1 to smithi11075562-14 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2796: finishing write tid 1 to smithi11075562-28 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2799: finishing rollback tid 2 to smithi11075562-14 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1820 (ObjNum 756 snap 244 seq_num 756) dirty exists 2024-09-14T03:47:18.010 INFO:tasks.rados.rados.0.smithi110.stdout:2796: finishing write tid 2 to smithi11075562-28 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:2795: finishing write tid 3 to smithi11075562-32 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1817 (ObjNum 875 snap 281 seq_num 875) dirty exists 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:2795: left oid 32 (ObjNum 875 snap 281 seq_num 875) 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:2800: finishing write tid 1 to smithi11075562-2 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:2796: finishing write tid 3 to smithi11075562-28 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 1652 (ObjNum 876 snap 281 seq_num 876) dirty exists 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:2796: left oid 28 (ObjNum 876 snap 281 seq_num 876) 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:2800: finishing write tid 2 to smithi11075562-2 2024-09-14T03:47:18.011 INFO:tasks.rados.rados.0.smithi110.stdout:2797: expect (ObjNum 796 snap 264 seq_num 796) 2024-09-14T03:47:18.120 INFO:tasks.rados.rados.0.smithi110.stdout:2800: finishing write tid 3 to smithi11075562-2 2024-09-14T03:47:18.120 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 1654 (ObjNum 877 snap 281 seq_num 877) dirty exists 2024-09-14T03:47:18.120 INFO:tasks.rados.rados.0.smithi110.stdout:2800: left oid 2 (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:47:18.120 INFO:tasks.rados.rados.0.smithi110.stdout:2795: done (7 left) 2024-09-14T03:47:18.120 INFO:tasks.rados.rados.0.smithi110.stdout:2796: done (6 left) 2024-09-14T03:47:18.120 INFO:tasks.rados.rados.0.smithi110.stdout:2797: done (5 left) 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2799: done (4 left) 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2800: done (3 left) 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2801: done (2 left) 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2804: snap_create 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2802: finishing write tid 1 to smithi11075562-42 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2802: finishing write tid 2 to smithi11075562-42 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2802: finishing write tid 3 to smithi11075562-42 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1831 (ObjNum 878 snap 281 seq_num 878) dirty exists 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2802: left oid 42 (ObjNum 878 snap 281 seq_num 878) 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2803: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:2803: finishing copy_from to smithi11075562-46 2024-09-14T03:47:18.121 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1821 (ObjNum 820 snap 272 seq_num 820) dirty exists 2024-09-14T03:47:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:17 smithi110 ceph-mon[86423]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T03:47:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:17 smithi110 ceph-mon[86423]: pgmap v582: 121 pgs: 121 active+clean; 286 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 139 KiB/s wr, 10 op/s 2024-09-14T03:47:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:17 smithi110 ceph-mon[88262]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T03:47:18.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:17 smithi110 ceph-mon[88262]: pgmap v582: 121 pgs: 121 active+clean; 286 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 139 KiB/s wr, 10 op/s 2024-09-14T03:47:18.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:17 smithi195 ceph-mon[82388]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T03:47:18.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:17 smithi195 ceph-mon[82388]: pgmap v582: 121 pgs: 121 active+clean; 286 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 139 KiB/s wr, 10 op/s 2024-09-14T03:47:18.768 INFO:tasks.rados.rados.0.smithi110.stdout:2802: done (2 left) 2024-09-14T03:47:18.768 INFO:tasks.rados.rados.0.smithi110.stdout:2803: done (1 left) 2024-09-14T03:47:18.768 INFO:tasks.rados.rados.0.smithi110.stdout:2804: done (0 left) 2024-09-14T03:47:18.768 INFO:tasks.rados.rados.0.smithi110.stdout:2805: snap_create 2024-09-14T03:47:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:18 smithi110 ceph-mon[86423]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T03:47:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:18 smithi110 ceph-mon[86423]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T03:47:19.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:18 smithi110 ceph-mon[88262]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T03:47:19.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:18 smithi110 ceph-mon[88262]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T03:47:19.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:18 smithi195 ceph-mon[82388]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T03:47:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:18 smithi195 ceph-mon[82388]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T03:47:19.769 INFO:tasks.rados.rados.0.smithi110.stdout:2805: done (0 left) 2024-09-14T03:47:19.770 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 283 2024-09-14T03:47:19.770 INFO:tasks.rados.rados.0.smithi110.stdout:2806: seq_num 879 ranges {8822784=327680} 2024-09-14T03:47:19.819 INFO:tasks.rados.rados.0.smithi110.stdout:2806: writing smithi11075562-42 from 8822784 to 9150464 tid 1 2024-09-14T03:47:19.819 INFO:tasks.rados.rados.0.smithi110.stdout:2807: read oid 35 snap 277 2024-09-14T03:47:19.819 INFO:tasks.rados.rados.0.smithi110.stdout:2807: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:47:19.819 INFO:tasks.rados.rados.0.smithi110.stdout:2808: delete oid 31 current snap is 283 2024-09-14T03:47:19.822 INFO:tasks.rados.rados.0.smithi110.stdout:2808: done (2 left) 2024-09-14T03:47:19.823 INFO:tasks.rados.rados.0.smithi110.stdout:2809: rmattr oid 3 current snap is 283 2024-09-14T03:47:19.823 INFO:tasks.rados.rados.0.smithi110.stdout:2809: done (2 left) 2024-09-14T03:47:19.823 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 283 2024-09-14T03:47:19.823 INFO:tasks.rados.rados.0.smithi110.stdout:2810: seq_num 881 ranges {9248768=360448} 2024-09-14T03:47:19.875 INFO:tasks.rados.rados.0.smithi110.stdout:2810: writing smithi11075562-26 from 9248768 to 9609216 tid 1 2024-09-14T03:47:19.875 INFO:tasks.rados.rados.0.smithi110.stdout:2811: rollback oid 8 current snap is 283 2024-09-14T03:47:19.875 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 8 to 252 2024-09-14T03:47:19.876 INFO:tasks.rados.rados.0.smithi110.stdout:2806: finishing write tid 1 to smithi11075562-42 2024-09-14T03:47:19.876 INFO:tasks.rados.rados.0.smithi110.stdout:2812: setattr oid 40 current snap is 283 2024-09-14T03:47:19.876 INFO:tasks.rados.rados.0.smithi110.stdout:2806: finishing write tid 2 to smithi11075562-42 2024-09-14T03:47:19.876 INFO:tasks.rados.rados.0.smithi110.stdout:2807: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:2806: finishing write tid 3 to smithi11075562-42 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1833 (ObjNum 879 snap 283 seq_num 879) dirty exists 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:2806: left oid 42 (ObjNum 879 snap 283 seq_num 879) 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:2811: finishing rollback tid 1 to smithi11075562-8 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 0 (ObjNum 65 snap 0 seq_num 43181008) dirty dne 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:2810: finishing write tid 1 to smithi11075562-26 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:2810: finishing write tid 2 to smithi11075562-26 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:2810: finishing write tid 3 to smithi11075562-26 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1823 (ObjNum 881 snap 283 seq_num 881) dirty exists 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:2810: left oid 26 (ObjNum 881 snap 283 seq_num 881) 2024-09-14T03:47:20.431 INFO:tasks.rados.rados.0.smithi110.stdout:2806: done (4 left) 2024-09-14T03:47:20.432 INFO:tasks.rados.rados.0.smithi110.stdout:2807: done (3 left) 2024-09-14T03:47:20.432 INFO:tasks.rados.rados.0.smithi110.stdout:2810: done (2 left) 2024-09-14T03:47:20.432 INFO:tasks.rados.rados.0.smithi110.stdout:2811: done (1 left) 2024-09-14T03:47:20.432 INFO:tasks.rados.rados.0.smithi110.stdout:2813: rollback oid 3 current snap is 283 2024-09-14T03:47:20.432 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 3 to 252 2024-09-14T03:47:20.432 INFO:tasks.rados.rados.0.smithi110.stdout:2814: snap_create 2024-09-14T03:47:20.433 INFO:tasks.rados.rados.0.smithi110.stdout:2813: finishing rollback tid 0 to smithi11075562-3 2024-09-14T03:47:20.433 INFO:tasks.rados.rados.0.smithi110.stdout:2813: finishing rollback tid 1 to smithi11075562-3 2024-09-14T03:47:20.433 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 1656 (ObjNum 33 snap 0 seq_num 2917113072) dirty dne 2024-09-14T03:47:20.434 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1520 (ObjNum 129 snap 0 seq_num 64784496) dirty exists 2024-09-14T03:47:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:47:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:47:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:47:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:2812: done (2 left) 2024-09-14T03:47:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:2813: done (1 left) 2024-09-14T03:47:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:2814: done (0 left) 2024-09-14T03:47:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:2815: rollback oid 14 current snap is 284 2024-09-14T03:47:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 14 to 283 2024-09-14T03:47:20.775 INFO:tasks.rados.rados.0.smithi110.stdout:2816: snap_create 2024-09-14T03:47:20.779 INFO:tasks.rados.rados.0.smithi110.stdout:2815: finishing rollback tid 0 to smithi11075562-14 2024-09-14T03:47:20.780 INFO:tasks.rados.rados.0.smithi110.stdout:2815: finishing rollback tid 1 to smithi11075562-14 2024-09-14T03:47:20.780 INFO:tasks.rados.rados.0.smithi110.stdout:2815: finishing rollback tid 2 to smithi11075562-14 2024-09-14T03:47:20.780 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1823 (ObjNum 756 snap 244 seq_num 756) dirty exists 2024-09-14T03:47:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:20 smithi110 ceph-mon[88262]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T03:47:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:20 smithi110 ceph-mon[88262]: pgmap v586: 121 pgs: 121 active+clean; 269 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 338 KiB/s wr, 13 op/s 2024-09-14T03:47:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:20 smithi110 ceph-mon[86423]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T03:47:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:20 smithi110 ceph-mon[86423]: pgmap v586: 121 pgs: 121 active+clean; 269 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 338 KiB/s wr, 13 op/s 2024-09-14T03:47:21.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:20 smithi195 ceph-mon[82388]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T03:47:21.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:20 smithi195 ceph-mon[82388]: pgmap v586: 121 pgs: 121 active+clean; 269 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 338 KiB/s wr, 13 op/s 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:2815: done (1 left) 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:2816: done (0 left) 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:2817: read oid 49 snap -1 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:2817: expect (ObjNum 869 snap 281 seq_num 869) 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:2818: rmattr oid 6 current snap is 285 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:2818: done (1 left) 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:2819: rollback oid 28 current snap is 285 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 28 to 279 2024-09-14T03:47:21.782 INFO:tasks.rados.rados.0.smithi110.stdout:2820: copy_from oid 34 from oid 35 current snap is 285 2024-09-14T03:47:21.783 INFO:tasks.rados.rados.0.smithi110.stdout:2821: delete oid 27 current snap is 285 2024-09-14T03:47:21.787 INFO:tasks.rados.rados.0.smithi110.stdout:2821: done (3 left) 2024-09-14T03:47:21.787 INFO:tasks.rados.rados.0.smithi110.stdout:2822: read oid 19 snap 252 2024-09-14T03:47:21.787 INFO:tasks.rados.rados.0.smithi110.stdout:2822: expect (ObjNum 778 snap 251 seq_num 778) 2024-09-14T03:47:21.788 INFO:tasks.rados.rados.0.smithi110.stdout:2823: snap_create 2024-09-14T03:47:21.788 INFO:tasks.rados.rados.0.smithi110.stdout:2820: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:47:21.789 INFO:tasks.rados.rados.0.smithi110.stdout:2819: finishing rollback tid 0 to smithi11075562-28 2024-09-14T03:47:21.790 INFO:tasks.rados.rados.0.smithi110.stdout:2819: finishing rollback tid 1 to smithi11075562-28 2024-09-14T03:47:21.790 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 1658 (ObjNum 129 snap 0 seq_num 363) dirty dne 2024-09-14T03:47:21.793 INFO:tasks.rados.rados.0.smithi110.stdout:2820: finishing copy_from to smithi11075562-34 2024-09-14T03:47:21.793 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1829 (ObjNum 65 snap 0 seq_num 1750750310) dirty exists 2024-09-14T03:47:21.804 INFO:tasks.rados.rados.0.smithi110.stdout:2822: expect (ObjNum 778 snap 251 seq_num 778) 2024-09-14T03:47:21.833 INFO:tasks.rados.rados.0.smithi110.stdout:2817: expect (ObjNum 869 snap 281 seq_num 869) 2024-09-14T03:47:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:21 smithi110 ceph-mon[88262]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T03:47:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:21 smithi110 ceph-mon[86423]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T03:47:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:47:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:47:21] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-14T03:47:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:21 smithi195 ceph-mon[82388]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T03:47:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:2817: done (4 left) 2024-09-14T03:47:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:2819: done (3 left) 2024-09-14T03:47:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:2820: done (2 left) 2024-09-14T03:47:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:2822: done (1 left) 2024-09-14T03:47:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:2823: done (0 left) 2024-09-14T03:47:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:2824: rollback oid 18 current snap is 286 2024-09-14T03:47:22.786 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 18 to 282 2024-09-14T03:47:22.787 INFO:tasks.rados.rados.0.smithi110.stdout:2825: copy_from oid 24 from oid 30 current snap is 286 2024-09-14T03:47:22.787 INFO:tasks.rados.rados.0.smithi110.stdout:2826: copy_from oid 10 from oid 22 current snap is 286 2024-09-14T03:47:22.787 INFO:tasks.rados.rados.0.smithi110.stdout:2827: snap_remove snap 277 2024-09-14T03:47:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:22 smithi110 ceph-mon[88262]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T03:47:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:22 smithi110 ceph-mon[88262]: pgmap v589: 121 pgs: 121 active+clean; 276 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:47:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:22 smithi110 ceph-mon[86423]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T03:47:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:22 smithi110 ceph-mon[86423]: pgmap v589: 121 pgs: 121 active+clean; 276 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:47:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:22 smithi195 ceph-mon[82388]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T03:47:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:22 smithi195 ceph-mon[82388]: pgmap v589: 121 pgs: 121 active+clean; 276 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 2.7 MiB/s wr, 20 op/s 2024-09-14T03:47:23.770 INFO:tasks.rados.rados.0.smithi110.stdout:2827: done (3 left) 2024-09-14T03:47:23.770 INFO:tasks.rados.rados.0.smithi110.stdout:2828: read oid 16 snap -1 2024-09-14T03:47:23.770 INFO:tasks.rados.rados.0.smithi110.stdout:2828: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:23.770 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 286 2024-09-14T03:47:23.771 INFO:tasks.rados.rados.0.smithi110.stdout:2829: seq_num 883 ranges {4317184=393216} 2024-09-14T03:47:23.797 INFO:tasks.rados.rados.0.smithi110.stdout:2829: writing smithi11075562-50 from 4317184 to 4710400 tid 1 2024-09-14T03:47:23.797 INFO:tasks.rados.rados.0.smithi110.stdout:2830: snap_remove snap 271 2024-09-14T03:47:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:23 smithi110 ceph-mon[86423]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T03:47:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:23 smithi110 ceph-mon[86423]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T03:47:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:23 smithi110 ceph-mon[88262]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T03:47:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:23 smithi110 ceph-mon[88262]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T03:47:24.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:23 smithi195 ceph-mon[82388]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T03:47:24.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:23 smithi195 ceph-mon[82388]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T03:47:24.794 INFO:tasks.rados.rados.0.smithi110.stdout:2830: done (5 left) 2024-09-14T03:47:24.794 INFO:tasks.rados.rados.0.smithi110.stdout:2831: copy_from oid 33 from oid 44 current snap is 286 2024-09-14T03:47:24.794 INFO:tasks.rados.rados.0.smithi110.stdout:2824: finishing rollback tid 0 to smithi11075562-18 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:2824: finishing rollback tid 1 to smithi11075562-18 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:2824: finishing rollback tid 2 to smithi11075562-18 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1523 (ObjNum 872 snap 281 seq_num 872) dirty exists 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:2826: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:2825: finishing copy_from racing read to smithi11075562-24 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:2825: finishing copy_from to smithi11075562-24 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1808 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:2826: finishing copy_from to smithi11075562-10 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1815 (ObjNum 862 snap 281 seq_num 862) dirty exists 2024-09-14T03:47:24.795 INFO:tasks.rados.rados.0.smithi110.stdout:2828: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:24.799 INFO:tasks.rados.rados.0.smithi110.stdout:2829: finishing write tid 1 to smithi11075562-50 2024-09-14T03:47:24.799 INFO:tasks.rados.rados.0.smithi110.stdout:2829: finishing write tid 2 to smithi11075562-50 2024-09-14T03:47:24.799 INFO:tasks.rados.rados.0.smithi110.stdout:2829: finishing write tid 3 to smithi11075562-50 2024-09-14T03:47:24.799 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 1499 (ObjNum 883 snap 286 seq_num 883) dirty exists 2024-09-14T03:47:24.799 INFO:tasks.rados.rados.0.smithi110.stdout:2829: left oid 50 (ObjNum 883 snap 286 seq_num 883) 2024-09-14T03:47:24.799 INFO:tasks.rados.rados.0.smithi110.stdout:2824: done (5 left) 2024-09-14T03:47:24.800 INFO:tasks.rados.rados.0.smithi110.stdout:2825: done (4 left) 2024-09-14T03:47:24.800 INFO:tasks.rados.rados.0.smithi110.stdout:2826: done (3 left) 2024-09-14T03:47:24.800 INFO:tasks.rados.rados.0.smithi110.stdout:2828: done (2 left) 2024-09-14T03:47:24.800 INFO:tasks.rados.rados.0.smithi110.stdout:2829: done (1 left) 2024-09-14T03:47:24.800 INFO:tasks.rados.rados.0.smithi110.stdout:2832: snap_remove snap 284 2024-09-14T03:47:25.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:24 smithi110 ceph-mon[86423]: pgmap v592: 121 pgs: 121 active+clean; 285 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.8 MiB/s wr, 17 op/s 2024-09-14T03:47:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:24 smithi110 ceph-mon[88262]: pgmap v592: 121 pgs: 121 active+clean; 285 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.8 MiB/s wr, 17 op/s 2024-09-14T03:47:25.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:24 smithi195 ceph-mon[82388]: pgmap v592: 121 pgs: 121 active+clean; 285 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.8 MiB/s wr, 17 op/s 2024-09-14T03:47:25.797 INFO:tasks.rados.rados.0.smithi110.stdout:2832: done (1 left) 2024-09-14T03:47:25.797 INFO:tasks.rados.rados.0.smithi110.stdout:2833: setattr oid 16 current snap is 286 2024-09-14T03:47:25.797 INFO:tasks.rados.rados.0.smithi110.stdout:2831: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:47:25.797 INFO:tasks.rados.rados.0.smithi110.stdout:2831: finishing copy_from to smithi11075562-33 2024-09-14T03:47:25.798 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 420 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:47:25.799 INFO:tasks.rados.rados.0.smithi110.stdout:2831: done (1 left) 2024-09-14T03:47:25.799 INFO:tasks.rados.rados.0.smithi110.stdout:2834: read oid 2 snap -1 2024-09-14T03:47:25.799 INFO:tasks.rados.rados.0.smithi110.stdout:2834: expect (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:47:25.799 INFO:tasks.rados.rados.0.smithi110.stdout:2835: delete oid 36 current snap is 286 2024-09-14T03:47:25.804 INFO:tasks.rados.rados.0.smithi110.stdout:2835: done (2 left) 2024-09-14T03:47:25.804 INFO:tasks.rados.rados.0.smithi110.stdout:2836: setattr oid 28 current snap is 286 2024-09-14T03:47:25.805 INFO:tasks.rados.rados.0.smithi110.stdout:2837: snap_create 2024-09-14T03:47:25.807 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 1524 (ObjNum 161 snap 0 seq_num 2917113200) dirty exists 2024-09-14T03:47:25.808 INFO:tasks.rados.rados.0.smithi110.stdout:2834: expect (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:47:25.817 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 1659 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:47:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:25 smithi110 ceph-mon[88262]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T03:47:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:25 smithi110 ceph-mon[86423]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T03:47:26.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:25 smithi195 ceph-mon[82388]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T03:47:26.803 INFO:tasks.rados.rados.0.smithi110.stdout:2833: done (3 left) 2024-09-14T03:47:26.804 INFO:tasks.rados.rados.0.smithi110.stdout:2834: done (2 left) 2024-09-14T03:47:26.804 INFO:tasks.rados.rados.0.smithi110.stdout:2836: done (1 left) 2024-09-14T03:47:26.804 INFO:tasks.rados.rados.0.smithi110.stdout:2837: done (0 left) 2024-09-14T03:47:26.804 INFO:tasks.rados.rados.0.smithi110.stdout:2838: snap_remove snap 285 2024-09-14T03:47:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:26 smithi110 ceph-mon[88262]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T03:47:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:26 smithi110 ceph-mon[86423]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T03:47:27.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:26 smithi195 ceph-mon[82388]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T03:47:27.806 INFO:tasks.rados.rados.0.smithi110.stdout:2838: done (0 left) 2024-09-14T03:47:27.806 INFO:tasks.rados.rados.0.smithi110.stdout:2839: snap_remove snap 270 2024-09-14T03:47:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:27 smithi110 ceph-mon[88262]: pgmap v595: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 295 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.3 MiB/s wr, 5 op/s 2024-09-14T03:47:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:27 smithi110 ceph-mon[88262]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T03:47:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:27 smithi110 ceph-mon[86423]: pgmap v595: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 295 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.3 MiB/s wr, 5 op/s 2024-09-14T03:47:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:27 smithi110 ceph-mon[86423]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T03:47:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:28.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:27 smithi195 ceph-mon[82388]: pgmap v595: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 295 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.3 MiB/s wr, 5 op/s 2024-09-14T03:47:28.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:27 smithi195 ceph-mon[82388]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T03:47:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:28.772 INFO:tasks.rados.rados.0.smithi110.stdout:2839: done (0 left) 2024-09-14T03:47:28.772 INFO:tasks.rados.rados.0.smithi110.stdout:2840: rollback oid 18 current snap is 287 2024-09-14T03:47:28.772 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 18 to 262 2024-09-14T03:47:28.773 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 287 2024-09-14T03:47:28.773 INFO:tasks.rados.rados.0.smithi110.stdout:2841: seq_num 885 ranges {4808704=425984} 2024-09-14T03:47:28.803 INFO:tasks.rados.rados.0.smithi110.stdout:2841: writing smithi11075562-10 from 4808704 to 5234688 tid 1 2024-09-14T03:47:28.803 INFO:tasks.rados.rados.0.smithi110.stdout:2842: snap_create 2024-09-14T03:47:28.803 INFO:tasks.rados.rados.0.smithi110.stdout:2840: finishing rollback tid 0 to smithi11075562-18 2024-09-14T03:47:28.803 INFO:tasks.rados.rados.0.smithi110.stdout:2840: finishing rollback tid 1 to smithi11075562-18 2024-09-14T03:47:28.803 INFO:tasks.rados.rados.0.smithi110.stdout:2840: finishing rollback tid 2 to smithi11075562-18 2024-09-14T03:47:28.803 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 1527 (ObjNum 1057 snap 0 seq_num 2917114064) dirty exists 2024-09-14T03:47:28.806 INFO:tasks.rados.rados.0.smithi110.stdout:2841: finishing write tid 1 to smithi11075562-10 2024-09-14T03:47:28.806 INFO:tasks.rados.rados.0.smithi110.stdout:2841: finishing write tid 2 to smithi11075562-10 2024-09-14T03:47:28.807 INFO:tasks.rados.rados.0.smithi110.stdout:2841: finishing write tid 3 to smithi11075562-10 2024-09-14T03:47:28.807 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1879 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:47:28.807 INFO:tasks.rados.rados.0.smithi110.stdout:2841: left oid 10 (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:47:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:28 smithi110 ceph-mon[88262]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T03:47:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:28 smithi110 ceph-mon[88262]: pgmap v598: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 295 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 3.0 MiB/s wr, 11 op/s 2024-09-14T03:47:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:28 smithi110 ceph-mon[88262]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T03:47:29.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:28 smithi110 ceph-mon[86423]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T03:47:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:28 smithi110 ceph-mon[86423]: pgmap v598: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 295 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 3.0 MiB/s wr, 11 op/s 2024-09-14T03:47:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:28 smithi110 ceph-mon[86423]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T03:47:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:28 smithi195 ceph-mon[82388]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T03:47:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:28 smithi195 ceph-mon[82388]: pgmap v598: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 295 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 3.0 MiB/s wr, 11 op/s 2024-09-14T03:47:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:28 smithi195 ceph-mon[82388]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T03:47:29.773 INFO:tasks.rados.rados.0.smithi110.stdout:2840: done (2 left) 2024-09-14T03:47:29.773 INFO:tasks.rados.rados.0.smithi110.stdout:2841: done (1 left) 2024-09-14T03:47:29.773 INFO:tasks.rados.rados.0.smithi110.stdout:2842: done (0 left) 2024-09-14T03:47:29.773 INFO:tasks.rados.rados.0.smithi110.stdout:2843: rollback oid 20 current snap is 288 2024-09-14T03:47:29.773 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 282 2024-09-14T03:47:29.773 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 288 2024-09-14T03:47:29.774 INFO:tasks.rados.rados.0.smithi110.stdout:2844: seq_num 886 ranges {0=442368} 2024-09-14T03:47:29.775 INFO:tasks.rados.rados.0.smithi110.stdout:2844: writing smithi11075562-35 from 0 to 442368 tid 1 2024-09-14T03:47:29.775 INFO:tasks.rados.rados.0.smithi110.stdout:2845: delete oid 5 current snap is 288 2024-09-14T03:47:29.780 INFO:tasks.rados.rados.0.smithi110.stdout:2843: finishing rollback tid 0 to smithi11075562-20 2024-09-14T03:47:29.781 INFO:tasks.rados.rados.0.smithi110.stdout:2843: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:47:29.782 INFO:tasks.rados.rados.0.smithi110.stdout:2843: finishing rollback tid 2 to smithi11075562-20 2024-09-14T03:47:29.782 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1824 (ObjNum 129 snap 0 seq_num 64548320) dirty exists 2024-09-14T03:47:29.782 INFO:tasks.rados.rados.0.smithi110.stdout:2844: finishing write tid 1 to smithi11075562-35 2024-09-14T03:47:29.782 INFO:tasks.rados.rados.0.smithi110.stdout:2844: finishing write tid 2 to smithi11075562-35 2024-09-14T03:47:29.782 INFO:tasks.rados.rados.0.smithi110.stdout:2843: done (2 left) 2024-09-14T03:47:29.782 INFO:tasks.rados.rados.0.smithi110.stdout:2845: done (1 left) 2024-09-14T03:47:29.782 INFO:tasks.rados.rados.0.smithi110.stdout:2846: setattr oid 3 current snap is 288 2024-09-14T03:47:29.784 INFO:tasks.rados.rados.0.smithi110.stdout:2844: finishing write tid 3 to smithi11075562-35 2024-09-14T03:47:29.784 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1835 (ObjNum 886 snap 288 seq_num 886) dirty exists 2024-09-14T03:47:29.784 INFO:tasks.rados.rados.0.smithi110.stdout:2844: left oid 35 (ObjNum 886 snap 288 seq_num 886) 2024-09-14T03:47:29.784 INFO:tasks.rados.rados.0.smithi110.stdout:2844: done (1 left) 2024-09-14T03:47:29.784 INFO:tasks.rados.rados.0.smithi110.stdout:2847: rmattr oid 27 current snap is 288 2024-09-14T03:47:29.785 INFO:tasks.rados.rados.0.smithi110.stdout:2847: done (1 left) 2024-09-14T03:47:29.785 INFO:tasks.rados.rados.0.smithi110.stdout:2848: rmattr oid 31 current snap is 288 2024-09-14T03:47:29.785 INFO:tasks.rados.rados.0.smithi110.stdout:2848: done (1 left) 2024-09-14T03:47:29.785 INFO:tasks.rados.rados.0.smithi110.stdout:2849: read oid 16 snap 275 2024-09-14T03:47:29.785 INFO:tasks.rados.rados.0.smithi110.stdout:2849: expect deleted 2024-09-14T03:47:29.785 INFO:tasks.rados.rados.0.smithi110.stdout:2850: read oid 15 snap -1 2024-09-14T03:47:29.785 INFO:tasks.rados.rados.0.smithi110.stdout:2850: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:29.785 INFO:tasks.rados.rados.0.smithi110.stdout:2851: delete oid 17 current snap is 288 2024-09-14T03:47:29.786 INFO:tasks.rados.rados.0.smithi110.stdout:2849: done (3 left) 2024-09-14T03:47:29.786 INFO:tasks.rados.rados.0.smithi110.stdout:2851: done (2 left) 2024-09-14T03:47:29.787 INFO:tasks.rados.rados.0.smithi110.stdout:2852: snap_remove snap 286 2024-09-14T03:47:30.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:47:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:47:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:47:30.778 INFO:tasks.rados.rados.0.smithi110.stdout:2852: done (2 left) 2024-09-14T03:47:30.778 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 288 2024-09-14T03:47:30.778 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 1660 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-14T03:47:30.779 INFO:tasks.rados.rados.0.smithi110.stdout:2853: seq_num 889 ranges {0=491520} 2024-09-14T03:47:30.781 INFO:tasks.rados.rados.0.smithi110.stdout:2853: writing smithi11075562-28 from 0 to 491520 tid 1 2024-09-14T03:47:30.781 INFO:tasks.rados.rados.0.smithi110.stdout:2846: done (2 left) 2024-09-14T03:47:30.782 INFO:tasks.rados.rados.0.smithi110.stdout:2854: read oid 23 snap -1 2024-09-14T03:47:30.782 INFO:tasks.rados.rados.0.smithi110.stdout:2854: expect deleted 2024-09-14T03:47:30.782 INFO:tasks.rados.rados.0.smithi110.stdout:2855: rmattr oid 37 current snap is 288 2024-09-14T03:47:30.782 INFO:tasks.rados.rados.0.smithi110.stdout:2855: done (3 left) 2024-09-14T03:47:30.782 INFO:tasks.rados.rados.0.smithi110.stdout:2856: snap_remove snap 251 2024-09-14T03:47:31.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:30 smithi110 ceph-mon[88262]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T03:47:31.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:30 smithi110 ceph-mon[86423]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T03:47:31.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:30 smithi195 ceph-mon[82388]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T03:47:31.780 INFO:tasks.rados.rados.0.smithi110.stdout:2856: done (3 left) 2024-09-14T03:47:31.780 INFO:tasks.rados.rados.0.smithi110.stdout:2857: snap_create 2024-09-14T03:47:31.780 INFO:tasks.rados.rados.0.smithi110.stdout:2850: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:31.781 INFO:tasks.rados.rados.0.smithi110.stdout:2853: finishing write tid 1 to smithi11075562-28 2024-09-14T03:47:31.781 INFO:tasks.rados.rados.0.smithi110.stdout:2853: finishing write tid 2 to smithi11075562-28 2024-09-14T03:47:31.781 INFO:tasks.rados.rados.0.smithi110.stdout:2853: finishing write tid 3 to smithi11075562-28 2024-09-14T03:47:31.781 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 1662 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-14T03:47:31.781 INFO:tasks.rados.rados.0.smithi110.stdout:2853: left oid 28 (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:47:31.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:47:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:47:31] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-14T03:47:31.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:31 smithi110 ceph-mon[86423]: pgmap v601: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 539 KiB/s wr, 16 op/s 2024-09-14T03:47:31.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:31 smithi110 ceph-mon[86423]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T03:47:31.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:31 smithi110 ceph-mon[88262]: pgmap v601: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 539 KiB/s wr, 16 op/s 2024-09-14T03:47:31.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:31 smithi110 ceph-mon[88262]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T03:47:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:31 smithi195 ceph-mon[82388]: pgmap v601: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 539 KiB/s wr, 16 op/s 2024-09-14T03:47:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:31 smithi195 ceph-mon[82388]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T03:47:32.781 INFO:tasks.rados.rados.0.smithi110.stdout:2850: done (3 left) 2024-09-14T03:47:32.781 INFO:tasks.rados.rados.0.smithi110.stdout:2853: done (2 left) 2024-09-14T03:47:32.782 INFO:tasks.rados.rados.0.smithi110.stdout:2854: done (1 left) 2024-09-14T03:47:32.782 INFO:tasks.rados.rados.0.smithi110.stdout:2857: done (0 left) 2024-09-14T03:47:32.782 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 289 2024-09-14T03:47:32.782 INFO:tasks.rados.rados.0.smithi110.stdout:2858: seq_num 890 ranges {0=507904} 2024-09-14T03:47:32.784 INFO:tasks.rados.rados.0.smithi110.stdout:2858: writing smithi11075562-20 from 0 to 507904 tid 1 2024-09-14T03:47:32.785 INFO:tasks.rados.rados.0.smithi110.stdout:2859: read oid 9 snap 252 2024-09-14T03:47:32.785 INFO:tasks.rados.rados.0.smithi110.stdout:2859: expect deleted 2024-09-14T03:47:32.785 INFO:tasks.rados.rados.0.smithi110.stdout:2860: read oid 30 snap -1 2024-09-14T03:47:32.785 INFO:tasks.rados.rados.0.smithi110.stdout:2860: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:47:32.785 INFO:tasks.rados.rados.0.smithi110.stdout:2861: snap_remove snap 275 2024-09-14T03:47:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:32 smithi110 ceph-mon[88262]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T03:47:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:32 smithi110 ceph-mon[88262]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T03:47:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:32 smithi110 ceph-mon[86423]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T03:47:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:32 smithi110 ceph-mon[86423]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T03:47:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:32 smithi195 ceph-mon[82388]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T03:47:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:32 smithi195 ceph-mon[82388]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T03:47:33.774 INFO:tasks.rados.rados.0.smithi110.stdout:2861: done (3 left) 2024-09-14T03:47:33.775 INFO:tasks.rados.rados.0.smithi110.stdout:2862: read oid 16 snap -1 2024-09-14T03:47:33.775 INFO:tasks.rados.rados.0.smithi110.stdout:2862: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:33.775 INFO:tasks.rados.rados.0.smithi110.stdout:2863: read oid 8 snap -1 2024-09-14T03:47:33.775 INFO:tasks.rados.rados.0.smithi110.stdout:2863: expect deleted 2024-09-14T03:47:33.775 INFO:tasks.rados.rados.0.smithi110.stdout:2864: snap_remove snap 282 2024-09-14T03:47:34.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:33 smithi110 ceph-mon[88262]: pgmap v604: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 219 KiB/s rd, 648 KiB/s wr, 9 op/s 2024-09-14T03:47:34.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:33 smithi110 ceph-mon[88262]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T03:47:34.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:33 smithi110 ceph-mon[86423]: pgmap v604: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 219 KiB/s rd, 648 KiB/s wr, 9 op/s 2024-09-14T03:47:34.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:33 smithi110 ceph-mon[86423]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T03:47:34.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:33 smithi195 ceph-mon[82388]: pgmap v604: 121 pgs: 121 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 219 KiB/s rd, 648 KiB/s wr, 9 op/s 2024-09-14T03:47:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:33 smithi195 ceph-mon[82388]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T03:47:34.774 INFO:tasks.rados.rados.0.smithi110.stdout:2864: done (5 left) 2024-09-14T03:47:34.775 INFO:tasks.rados.rados.0.smithi110.stdout:2865: rollback oid 50 current snap is 289 2024-09-14T03:47:34.775 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 252 2024-09-14T03:47:34.775 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 289 2024-09-14T03:47:34.775 INFO:tasks.rados.rados.0.smithi110.stdout:2866: seq_num 891 ranges {139264=532480} 2024-09-14T03:47:34.779 INFO:tasks.rados.rados.0.smithi110.stdout:2866: writing smithi11075562-12 from 139264 to 671744 tid 1 2024-09-14T03:47:34.779 INFO:tasks.rados.rados.0.smithi110.stdout:2858: finishing write tid 1 to smithi11075562-20 2024-09-14T03:47:34.779 INFO:tasks.rados.rados.0.smithi110.stdout:2858: finishing write tid 2 to smithi11075562-20 2024-09-14T03:47:34.779 INFO:tasks.rados.rados.0.smithi110.stdout:2858: finishing write tid 3 to smithi11075562-20 2024-09-14T03:47:34.779 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1826 (ObjNum 890 snap 289 seq_num 890) dirty exists 2024-09-14T03:47:34.779 INFO:tasks.rados.rados.0.smithi110.stdout:2858: left oid 20 (ObjNum 890 snap 289 seq_num 890) 2024-09-14T03:47:34.779 INFO:tasks.rados.rados.0.smithi110.stdout:2860: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:47:35.339 INFO:tasks.rados.rados.0.smithi110.stdout:2862: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:2865: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:2865: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:2865: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 1502 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:2866: finishing write tid 1 to smithi11075562-12 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:2866: finishing write tid 2 to smithi11075562-12 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:2866: finishing write tid 3 to smithi11075562-12 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 1986 (ObjNum 891 snap 289 seq_num 891) dirty exists 2024-09-14T03:47:35.346 INFO:tasks.rados.rados.0.smithi110.stdout:2866: left oid 12 (ObjNum 891 snap 289 seq_num 891) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2858: done (6 left) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2859: done (5 left) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2860: done (4 left) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2862: done (3 left) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2863: done (2 left) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2865: done (1 left) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2866: done (0 left) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2867: read oid 49 snap 262 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2867: expect (ObjNum 784 snap 257 seq_num 784) 2024-09-14T03:47:35.347 INFO:tasks.rados.rados.0.smithi110.stdout:2868: delete oid 5 current snap is 289 2024-09-14T03:47:35.349 INFO:tasks.rados.rados.0.smithi110.stdout:2868: done (1 left) 2024-09-14T03:47:35.349 INFO:tasks.rados.rados.0.smithi110.stdout:2869: copy_from oid 41 from oid 6 current snap is 289 2024-09-14T03:47:35.349 INFO:tasks.rados.rados.0.smithi110.stdout:2870: copy_from oid 36 from oid 40 current snap is 289 2024-09-14T03:47:35.349 INFO:tasks.rados.rados.0.smithi110.stdout:2871: snap_create 2024-09-14T03:47:35.354 INFO:tasks.rados.rados.0.smithi110.stdout:2870: finishing copy_from racing read to smithi11075562-36 2024-09-14T03:47:35.354 INFO:tasks.rados.rados.0.smithi110.stdout:2870: finishing copy_from to smithi11075562-36 2024-09-14T03:47:35.355 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 1520 (ObjNum 97 snap 0 seq_num 66791584) dirty exists 2024-09-14T03:47:35.370 INFO:tasks.rados.rados.0.smithi110.stdout:2867: expect (ObjNum 784 snap 257 seq_num 784) 2024-09-14T03:47:35.615 INFO:tasks.rados.rados.0.smithi110.stdout:2869: finishing copy_from racing read to smithi11075562-41 2024-09-14T03:47:35.616 INFO:tasks.rados.rados.0.smithi110.stdout:2869: finishing copy_from to smithi11075562-41 2024-09-14T03:47:35.616 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1506 (ObjNum 828 snap 274 seq_num 828) dirty exists 2024-09-14T03:47:35.795 INFO:tasks.rados.rados.0.smithi110.stdout:2867: done (3 left) 2024-09-14T03:47:35.795 INFO:tasks.rados.rados.0.smithi110.stdout:2869: done (2 left) 2024-09-14T03:47:35.795 INFO:tasks.rados.rados.0.smithi110.stdout:2870: done (1 left) 2024-09-14T03:47:35.795 INFO:tasks.rados.rados.0.smithi110.stdout:2871: done (0 left) 2024-09-14T03:47:35.795 INFO:tasks.rados.rados.0.smithi110.stdout:2872: copy_from oid 4 from oid 16 current snap is 290 2024-09-14T03:47:35.795 INFO:tasks.rados.rados.0.smithi110.stdout:2873: snap_create 2024-09-14T03:47:35.797 INFO:tasks.rados.rados.0.smithi110.stdout:2872: finishing copy_from racing read to smithi11075562-4 2024-09-14T03:47:35.800 INFO:tasks.rados.rados.0.smithi110.stdout:2872: finishing copy_from to smithi11075562-4 2024-09-14T03:47:35.800 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 1524 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-14T03:47:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:35 smithi110 ceph-mon[88262]: pgmap v607: 121 pgs: 121 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 232 KiB/s wr, 2 op/s 2024-09-14T03:47:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:35 smithi110 ceph-mon[88262]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T03:47:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:35 smithi110 ceph-mon[86423]: pgmap v607: 121 pgs: 121 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 232 KiB/s wr, 2 op/s 2024-09-14T03:47:36.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:35 smithi110 ceph-mon[86423]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T03:47:36.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:35 smithi195 ceph-mon[82388]: pgmap v607: 121 pgs: 121 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 232 KiB/s wr, 2 op/s 2024-09-14T03:47:36.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:35 smithi195 ceph-mon[82388]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T03:47:36.801 INFO:tasks.rados.rados.0.smithi110.stdout:2872: done (1 left) 2024-09-14T03:47:36.801 INFO:tasks.rados.rados.0.smithi110.stdout:2873: done (0 left) 2024-09-14T03:47:36.802 INFO:tasks.rados.rados.0.smithi110.stdout:2874: read oid 49 snap -1 2024-09-14T03:47:36.802 INFO:tasks.rados.rados.0.smithi110.stdout:2874: expect (ObjNum 869 snap 281 seq_num 869) 2024-09-14T03:47:36.802 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 291 2024-09-14T03:47:36.802 INFO:tasks.rados.rados.0.smithi110.stdout:2875: seq_num 893 ranges {4808704=565248} 2024-09-14T03:47:36.832 INFO:tasks.rados.rados.0.smithi110.stdout:2875: writing smithi11075562-22 from 4808704 to 5373952 tid 1 2024-09-14T03:47:36.832 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 291 2024-09-14T03:47:36.832 INFO:tasks.rados.rados.0.smithi110.stdout:2876: seq_num 894 ranges {9609216=581632} 2024-09-14T03:47:36.886 INFO:tasks.rados.rados.0.smithi110.stdout:2876: writing smithi11075562-26 from 9609216 to 10190848 tid 1 2024-09-14T03:47:36.886 INFO:tasks.rados.rados.0.smithi110.stdout:2877: copy_from oid 45 from oid 28 current snap is 291 2024-09-14T03:47:36.886 INFO:tasks.rados.rados.0.smithi110.stdout:2874: expect (ObjNum 869 snap 281 seq_num 869) 2024-09-14T03:47:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:36 smithi110 ceph-mon[88262]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T03:47:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:36 smithi110 ceph-mon[86423]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T03:47:37.208 INFO:tasks.rados.rados.0.smithi110.stdout:2875: finishing write tid 1 to smithi11075562-22 2024-09-14T03:47:37.208 INFO:tasks.rados.rados.0.smithi110.stdout:2875: finishing write tid 2 to smithi11075562-22 2024-09-14T03:47:37.208 INFO:tasks.rados.rados.0.smithi110.stdout:2875: finishing write tid 3 to smithi11075562-22 2024-09-14T03:47:37.208 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1819 (ObjNum 893 snap 291 seq_num 893) dirty exists 2024-09-14T03:47:37.208 INFO:tasks.rados.rados.0.smithi110.stdout:2875: left oid 22 (ObjNum 893 snap 291 seq_num 893) 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2876: finishing write tid 1 to smithi11075562-26 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2876: finishing write tid 2 to smithi11075562-26 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2876: finishing write tid 3 to smithi11075562-26 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1825 (ObjNum 894 snap 291 seq_num 894) dirty exists 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2876: left oid 26 (ObjNum 894 snap 291 seq_num 894) 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2877: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2877: finishing copy_from to smithi11075562-45 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 1662 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2874: done (3 left) 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2875: done (2 left) 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2876: done (1 left) 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2877: done (0 left) 2024-09-14T03:47:37.209 INFO:tasks.rados.rados.0.smithi110.stdout:2878: snap_create 2024-09-14T03:47:37.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:36 smithi195 ceph-mon[82388]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T03:47:37.805 INFO:tasks.rados.rados.0.smithi110.stdout:2878: done (0 left) 2024-09-14T03:47:37.806 INFO:tasks.rados.rados.0.smithi110.stdout:2879: snap_remove snap 288 2024-09-14T03:47:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:37 smithi110 ceph-mon[88262]: pgmap v610: 121 pgs: 121 active+clean; 300 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 374 KiB/s wr, 10 op/s 2024-09-14T03:47:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:37 smithi110 ceph-mon[88262]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T03:47:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:37 smithi110 ceph-mon[86423]: pgmap v610: 121 pgs: 121 active+clean; 300 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 374 KiB/s wr, 10 op/s 2024-09-14T03:47:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:37 smithi110 ceph-mon[86423]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T03:47:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:37 smithi195 ceph-mon[82388]: pgmap v610: 121 pgs: 121 active+clean; 300 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 374 KiB/s wr, 10 op/s 2024-09-14T03:47:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:37 smithi195 ceph-mon[82388]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T03:47:38.775 INFO:tasks.rados.rados.0.smithi110.stdout:2879: done (0 left) 2024-09-14T03:47:38.775 INFO:tasks.rados.rados.0.smithi110.stdout:2880: snap_remove snap 261 2024-09-14T03:47:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:38 smithi110 ceph-mon[88262]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T03:47:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:38 smithi110 ceph-mon[88262]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T03:47:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:38 smithi110 ceph-mon[86423]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T03:47:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:38 smithi110 ceph-mon[86423]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T03:47:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:38 smithi195 ceph-mon[82388]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T03:47:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:38 smithi195 ceph-mon[82388]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T03:47:39.779 INFO:tasks.rados.rados.0.smithi110.stdout:2880: done (0 left) 2024-09-14T03:47:39.779 INFO:tasks.rados.rados.0.smithi110.stdout:2881: rollback oid 35 current snap is 292 2024-09-14T03:47:39.780 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 281 2024-09-14T03:47:39.780 INFO:tasks.rados.rados.0.smithi110.stdout:2882: snap_remove snap 287 2024-09-14T03:47:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:39 smithi110 ceph-mon[88262]: pgmap v613: 121 pgs: 121 active+clean; 303 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2024-09-14T03:47:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:39 smithi110 ceph-mon[88262]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T03:47:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:39 smithi110 ceph-mon[86423]: pgmap v613: 121 pgs: 121 active+clean; 303 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2024-09-14T03:47:40.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:39 smithi110 ceph-mon[86423]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T03:47:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:39 smithi195 ceph-mon[82388]: pgmap v613: 121 pgs: 121 active+clean; 303 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2024-09-14T03:47:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:39 smithi195 ceph-mon[82388]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T03:47:40.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:47:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:47:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:47:40.779 INFO:tasks.rados.rados.0.smithi110.stdout:2882: done (1 left) 2024-09-14T03:47:40.779 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 292 2024-09-14T03:47:40.779 INFO:tasks.rados.rados.0.smithi110.stdout:2883: seq_num 895 ranges {106496=598016} 2024-09-14T03:47:40.783 INFO:tasks.rados.rados.0.smithi110.stdout:2883: writing smithi11075562-13 from 106496 to 704512 tid 1 2024-09-14T03:47:40.783 INFO:tasks.rados.rados.0.smithi110.stdout:2884: setattr oid 41 current snap is 292 2024-09-14T03:47:40.783 INFO:tasks.rados.rados.0.smithi110.stdout:2881: finishing rollback tid 0 to smithi11075562-35 2024-09-14T03:47:40.783 INFO:tasks.rados.rados.0.smithi110.stdout:2881: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:47:40.784 INFO:tasks.rados.rados.0.smithi110.stdout:2881: finishing rollback tid 2 to smithi11075562-35 2024-09-14T03:47:40.784 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1838 (ObjNum 65 snap 0 seq_num 1667979865) dirty exists 2024-09-14T03:47:40.788 INFO:tasks.rados.rados.0.smithi110.stdout:2881: done (2 left) 2024-09-14T03:47:40.788 INFO:tasks.rados.rados.0.smithi110.stdout:2885: read oid 49 snap -1 2024-09-14T03:47:40.788 INFO:tasks.rados.rados.0.smithi110.stdout:2885: expect (ObjNum 869 snap 281 seq_num 869) 2024-09-14T03:47:40.788 INFO:tasks.rados.rados.0.smithi110.stdout:2886: rollback oid 24 current snap is 292 2024-09-14T03:47:40.789 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 290 2024-09-14T03:47:40.789 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 292 2024-09-14T03:47:40.789 INFO:tasks.rados.rados.0.smithi110.stdout:2887: seq_num 896 ranges {0=614400} 2024-09-14T03:47:40.793 INFO:tasks.rados.rados.0.smithi110.stdout:2887: writing smithi11075562-8 from 0 to 614400 tid 1 2024-09-14T03:47:40.793 INFO:tasks.rados.rados.0.smithi110.stdout:2888: read oid 50 snap -1 2024-09-14T03:47:40.793 INFO:tasks.rados.rados.0.smithi110.stdout:2888: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:47:40.793 INFO:tasks.rados.rados.0.smithi110.stdout:2889: read oid 30 snap -1 2024-09-14T03:47:40.793 INFO:tasks.rados.rados.0.smithi110.stdout:2889: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:47:40.793 INFO:tasks.rados.rados.0.smithi110.stdout:2890: delete oid 11 current snap is 292 2024-09-14T03:47:40.793 INFO:tasks.rados.rados.0.smithi110.stdout:2883: finishing write tid 1 to smithi11075562-13 2024-09-14T03:47:40.794 INFO:tasks.rados.rados.0.smithi110.stdout:2883: finishing write tid 2 to smithi11075562-13 2024-09-14T03:47:40.795 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1824 (ObjNum 828 snap 274 seq_num 828) dirty exists 2024-09-14T03:47:40.797 INFO:tasks.rados.rados.0.smithi110.stdout:2884: done (7 left) 2024-09-14T03:47:40.797 INFO:tasks.rados.rados.0.smithi110.stdout:2890: done (6 left) 2024-09-14T03:47:40.797 INFO:tasks.rados.rados.0.smithi110.stdout:2891: snap_create 2024-09-14T03:47:40.799 INFO:tasks.rados.rados.0.smithi110.stdout:2886: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:47:40.800 INFO:tasks.rados.rados.0.smithi110.stdout:2886: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:47:40.801 INFO:tasks.rados.rados.0.smithi110.stdout:2886: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:47:40.801 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1882 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:47:40.823 INFO:tasks.rados.rados.0.smithi110.stdout:2887: finishing write tid 1 to smithi11075562-8 2024-09-14T03:47:40.823 INFO:tasks.rados.rados.0.smithi110.stdout:2887: finishing write tid 2 to smithi11075562-8 2024-09-14T03:47:40.824 INFO:tasks.rados.rados.0.smithi110.stdout:2885: expect (ObjNum 869 snap 281 seq_num 869) 2024-09-14T03:47:41.154 INFO:tasks.rados.rados.0.smithi110.stdout:2883: finishing write tid 3 to smithi11075562-13 2024-09-14T03:47:41.154 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1840 (ObjNum 895 snap 292 seq_num 895) dirty exists 2024-09-14T03:47:41.154 INFO:tasks.rados.rados.0.smithi110.stdout:2883: left oid 13 (ObjNum 895 snap 292 seq_num 895) 2024-09-14T03:47:41.154 INFO:tasks.rados.rados.0.smithi110.stdout:2889: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:47:41.711 INFO:tasks.rados.rados.0.smithi110.stdout:2887: finishing write tid 3 to smithi11075562-8 2024-09-14T03:47:41.711 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1821 (ObjNum 896 snap 292 seq_num 896) dirty exists 2024-09-14T03:47:41.712 INFO:tasks.rados.rados.0.smithi110.stdout:2887: left oid 8 (ObjNum 896 snap 292 seq_num 896) 2024-09-14T03:47:41.712 INFO:tasks.rados.rados.0.smithi110.stdout:2888: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:47:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:47:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:47:41] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-14T03:47:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:41 smithi110 ceph-mon[88262]: pgmap v616: 121 pgs: 121 active+clean; 301 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 956 KiB/s wr, 3 op/s 2024-09-14T03:47:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:41 smithi110 ceph-mon[88262]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T03:47:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:41 smithi110 ceph-mon[86423]: pgmap v616: 121 pgs: 121 active+clean; 301 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 956 KiB/s wr, 3 op/s 2024-09-14T03:47:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:41 smithi110 ceph-mon[86423]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T03:47:42.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:41 smithi195 ceph-mon[82388]: pgmap v616: 121 pgs: 121 active+clean; 301 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 956 KiB/s wr, 3 op/s 2024-09-14T03:47:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:41 smithi195 ceph-mon[82388]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T03:47:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:42 smithi195 ceph-mon[82388]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T03:47:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:43.145 INFO:tasks.rados.rados.0.smithi110.stdout:2883: done (6 left) 2024-09-14T03:47:43.145 INFO:tasks.rados.rados.0.smithi110.stdout:2885: done (5 left) 2024-09-14T03:47:43.145 INFO:tasks.rados.rados.0.smithi110.stdout:2886: done (4 left) 2024-09-14T03:47:43.145 INFO:tasks.rados.rados.0.smithi110.stdout:2887: done (3 left) 2024-09-14T03:47:43.145 INFO:tasks.rados.rados.0.smithi110.stdout:2888: done (2 left) 2024-09-14T03:47:43.145 INFO:tasks.rados.rados.0.smithi110.stdout:2889: done (1 left) 2024-09-14T03:47:43.146 INFO:tasks.rados.rados.0.smithi110.stdout:2891: done (0 left) 2024-09-14T03:47:43.146 INFO:tasks.rados.rados.0.smithi110.stdout:2892: copy_from oid 49 from oid 26 current snap is 293 2024-09-14T03:47:43.146 INFO:tasks.rados.rados.0.smithi110.stdout:2893: snap_create 2024-09-14T03:47:43.184 INFO:tasks.rados.rados.0.smithi110.stdout:2892: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:47:43.187 INFO:tasks.rados.rados.0.smithi110.stdout:2892: finishing copy_from to smithi11075562-49 2024-09-14T03:47:43.187 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1825 (ObjNum 894 snap 291 seq_num 894) dirty exists 2024-09-14T03:47:43.195 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:42 smithi110 ceph-mon[88262]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T03:47:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:42 smithi110 ceph-mon[86423]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T03:47:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:43.774 INFO:tasks.rados.rados.0.smithi110.stdout:2892: done (1 left) 2024-09-14T03:47:43.775 INFO:tasks.rados.rados.0.smithi110.stdout:2893: done (0 left) 2024-09-14T03:47:43.775 INFO:tasks.rados.rados.0.smithi110.stdout:2894: delete oid 22 current snap is 294 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:2894: done (0 left) 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:2895: rollback oid 19 current snap is 294 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 293 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:2896: rmattr oid 23 current snap is 294 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:2896: done (1 left) 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:2897: read oid 50 snap -1 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:2897: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 294 2024-09-14T03:47:43.778 INFO:tasks.rados.rados.0.smithi110.stdout:2898: seq_num 899 ranges {9150464=663552} 2024-09-14T03:47:43.832 INFO:tasks.rados.rados.0.smithi110.stdout:2898: writing smithi11075562-42 from 9150464 to 9814016 tid 1 2024-09-14T03:47:43.833 INFO:tasks.rados.rados.0.smithi110.stdout:2899: snap_create 2024-09-14T03:47:43.833 INFO:tasks.rados.rados.0.smithi110.stdout:2895: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:47:43.833 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 1517 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-14T03:47:43.833 INFO:tasks.rados.rados.0.smithi110.stdout:2897: expect (ObjNum 677 snap 211 seq_num 677) 2024-09-14T03:47:43.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:43 smithi110 ceph-mon[88262]: pgmap v619: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 546 KiB/s wr, 5 op/s 2024-09-14T03:47:43.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:43 smithi110 ceph-mon[88262]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T03:47:43.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:43 smithi110 ceph-mon[86423]: pgmap v619: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 546 KiB/s wr, 5 op/s 2024-09-14T03:47:43.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:43 smithi110 ceph-mon[86423]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T03:47:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:43 smithi195 ceph-mon[82388]: pgmap v619: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 546 KiB/s wr, 5 op/s 2024-09-14T03:47:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:43 smithi195 ceph-mon[82388]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T03:47:45.275 INFO:tasks.rados.rados.0.smithi110.stdout:2898: finishing write tid 1 to smithi11075562-42 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:2898: finishing write tid 2 to smithi11075562-42 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:2898: finishing write tid 3 to smithi11075562-42 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1842 (ObjNum 899 snap 294 seq_num 899) dirty exists 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:2898: left oid 42 (ObjNum 899 snap 294 seq_num 899) 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:2895: done (3 left) 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:2897: done (2 left) 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:2898: done (1 left) 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:2899: done (0 left) 2024-09-14T03:47:45.276 INFO:tasks.rados.rados.0.smithi110.stdout:2900: delete oid 33 current snap is 295 2024-09-14T03:47:45.278 INFO:tasks.rados.rados.0.smithi110.stdout:2900: done (0 left) 2024-09-14T03:47:45.279 INFO:tasks.rados.rados.0.smithi110.stdout:2901: snap_remove snap 262 2024-09-14T03:47:45.801 INFO:tasks.rados.rados.0.smithi110.stdout:2901: done (0 left) 2024-09-14T03:47:45.801 INFO:tasks.rados.rados.0.smithi110.stdout:2902: snap_remove snap 281 2024-09-14T03:47:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:45 smithi110 ceph-mon[88262]: pgmap v621: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 541 KiB/s wr, 10 op/s 2024-09-14T03:47:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:45 smithi110 ceph-mon[88262]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T03:47:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:45 smithi110 ceph-mon[86423]: pgmap v621: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 541 KiB/s wr, 10 op/s 2024-09-14T03:47:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:45 smithi110 ceph-mon[86423]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T03:47:46.209 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:47:45 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=infra.usagestats t=2024-09-14T03:47:45.734018385Z level=info msg="Usage stats are ready to report" 2024-09-14T03:47:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:45 smithi195 ceph-mon[82388]: pgmap v621: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 541 KiB/s wr, 10 op/s 2024-09-14T03:47:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:45 smithi195 ceph-mon[82388]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T03:47:46.815 INFO:tasks.rados.rados.0.smithi110.stdout:2902: done (0 left) 2024-09-14T03:47:46.815 INFO:tasks.rados.rados.0.smithi110.stdout:2903: read oid 40 snap -1 2024-09-14T03:47:46.815 INFO:tasks.rados.rados.0.smithi110.stdout:2903: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:46.816 INFO:tasks.rados.rados.0.smithi110.stdout:2904: read oid 42 snap 279 2024-09-14T03:47:46.816 INFO:tasks.rados.rados.0.smithi110.stdout:2904: expect (ObjNum 842 snap 275 seq_num 842) 2024-09-14T03:47:46.816 INFO:tasks.rados.rados.0.smithi110.stdout:2905: snap_create 2024-09-14T03:47:46.821 INFO:tasks.rados.rados.0.smithi110.stdout:2903: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:46.862 INFO:tasks.rados.rados.0.smithi110.stdout:2904: expect (ObjNum 842 snap 275 seq_num 842) 2024-09-14T03:47:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:46 smithi110 ceph-mon[88262]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T03:47:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:46 smithi110 ceph-mon[86423]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T03:47:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:46 smithi195 ceph-mon[82388]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T03:47:47.811 INFO:tasks.rados.rados.0.smithi110.stdout:2903: done (2 left) 2024-09-14T03:47:47.811 INFO:tasks.rados.rados.0.smithi110.stdout:2904: done (1 left) 2024-09-14T03:47:47.811 INFO:tasks.rados.rados.0.smithi110.stdout:2905: done (0 left) 2024-09-14T03:47:47.811 INFO:tasks.rados.rados.0.smithi110.stdout:2906: copy_from oid 43 from oid 41 current snap is 296 2024-09-14T03:47:47.811 INFO:tasks.rados.rados.0.smithi110.stdout:2907: delete oid 31 current snap is 296 2024-09-14T03:47:47.815 INFO:tasks.rados.rados.0.smithi110.stdout:2907: done (1 left) 2024-09-14T03:47:47.815 INFO:tasks.rados.rados.0.smithi110.stdout:2908: copy_from oid 10 from oid 31 current snap is 296 2024-09-14T03:47:47.816 INFO:tasks.rados.rados.0.smithi110.stdout:2909: read oid 4 snap -1 2024-09-14T03:47:47.816 INFO:tasks.rados.rados.0.smithi110.stdout:2909: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:47.816 INFO:tasks.rados.rados.0.smithi110.stdout:2910: rollback oid 22 current snap is 296 2024-09-14T03:47:47.816 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 22 to 252 2024-09-14T03:47:47.816 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 296 2024-09-14T03:47:47.816 INFO:tasks.rados.rados.0.smithi110.stdout:2911: seq_num 902 ranges {3350528=712704} 2024-09-14T03:47:47.838 INFO:tasks.rados.rados.0.smithi110.stdout:2911: writing smithi11075562-6 from 3350528 to 4063232 tid 1 2024-09-14T03:47:47.839 INFO:tasks.rados.rados.0.smithi110.stdout:2912: delete oid 21 current snap is 296 2024-09-14T03:47:47.839 INFO:tasks.rados.rados.0.smithi110.stdout:2909: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:47.847 INFO:tasks.rados.rados.0.smithi110.stdout:2908: finishing copy_from to smithi11075562-10 2024-09-14T03:47:47.847 INFO:tasks.rados.rados.0.smithi110.stdout:2908: got expected ENOENT (src dne) 2024-09-14T03:47:47.847 INFO:tasks.rados.rados.0.smithi110.stdout:2908: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:47:47.847 INFO:tasks.rados.rados.0.smithi110.stdout:2910: finishing rollback tid 1 to smithi11075562-22 2024-09-14T03:47:47.847 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1822 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:47:47.847 INFO:tasks.rados.rados.0.smithi110.stdout:2906: finishing copy_from racing read to smithi11075562-43 2024-09-14T03:47:47.847 INFO:tasks.rados.rados.0.smithi110.stdout:2906: finishing copy_from to smithi11075562-43 2024-09-14T03:47:47.847 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1824 (ObjNum 828 snap 274 seq_num 828) dirty exists 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2911: finishing write tid 1 to smithi11075562-6 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2911: finishing write tid 2 to smithi11075562-6 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2911: finishing write tid 3 to smithi11075562-6 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1529 (ObjNum 902 snap 296 seq_num 902) dirty exists 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2911: left oid 6 (ObjNum 902 snap 296 seq_num 902) 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2906: done (5 left) 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2908: done (4 left) 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2909: done (3 left) 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2910: done (2 left) 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2911: done (1 left) 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2912: done (0 left) 2024-09-14T03:47:47.848 INFO:tasks.rados.rados.0.smithi110.stdout:2913: snap_create 2024-09-14T03:47:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:47 smithi110 ceph-mon[88262]: pgmap v624: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 149 KiB/s wr, 9 op/s 2024-09-14T03:47:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:47 smithi110 ceph-mon[88262]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T03:47:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:47 smithi110 ceph-mon[86423]: pgmap v624: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 149 KiB/s wr, 9 op/s 2024-09-14T03:47:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:47 smithi110 ceph-mon[86423]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T03:47:48.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:47 smithi195 ceph-mon[82388]: pgmap v624: 121 pgs: 121 active+clean; 303 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 149 KiB/s wr, 9 op/s 2024-09-14T03:47:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:47 smithi195 ceph-mon[82388]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T03:47:48.777 INFO:tasks.rados.rados.0.smithi110.stdout:2913: done (0 left) 2024-09-14T03:47:48.777 INFO:tasks.rados.rados.0.smithi110.stdout:2914: read oid 11 snap -1 2024-09-14T03:47:48.777 INFO:tasks.rados.rados.0.smithi110.stdout:2914: expect deleted 2024-09-14T03:47:48.777 INFO:tasks.rados.rados.0.smithi110.stdout:2915: snap_remove snap 252 2024-09-14T03:47:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:48 smithi110 ceph-mon[86423]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T03:47:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:48 smithi110 ceph-mon[86423]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T03:47:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:48 smithi110 ceph-mon[88262]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T03:47:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:48 smithi110 ceph-mon[88262]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T03:47:49.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:48 smithi195 ceph-mon[82388]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T03:47:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:48 smithi195 ceph-mon[82388]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T03:47:49.779 INFO:tasks.rados.rados.0.smithi110.stdout:2915: done (1 left) 2024-09-14T03:47:49.780 INFO:tasks.rados.rados.0.smithi110.stdout:2916: setattr oid 1 current snap is 297 2024-09-14T03:47:49.783 INFO:tasks.rados.rados.0.smithi110.stdout:2914: done (1 left) 2024-09-14T03:47:49.783 INFO:tasks.rados.rados.0.smithi110.stdout:2917: read oid 46 snap -1 2024-09-14T03:47:49.783 INFO:tasks.rados.rados.0.smithi110.stdout:2917: expect (ObjNum 820 snap 272 seq_num 820) 2024-09-14T03:47:49.784 INFO:tasks.rados.rados.0.smithi110.stdout:2918: rollback oid 45 current snap is 297 2024-09-14T03:47:49.784 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 293 2024-09-14T03:47:49.784 INFO:tasks.rados.rados.0.smithi110.stdout:2919: copy_from oid 49 from oid 12 current snap is 297 2024-09-14T03:47:49.784 INFO:tasks.rados.rados.0.smithi110.stdout:2920: rollback oid 50 current snap is 297 2024-09-14T03:47:49.784 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 50 to 290 2024-09-14T03:47:49.784 INFO:tasks.rados.rados.0.smithi110.stdout:2921: copy_from oid 20 from oid 41 current snap is 297 2024-09-14T03:47:49.785 INFO:tasks.rados.rados.0.smithi110.stdout:2922: delete oid 33 current snap is 297 2024-09-14T03:47:49.790 INFO:tasks.rados.rados.0.smithi110.stdout:2918: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:47:49.796 INFO:tasks.rados.rados.0.smithi110.stdout:2920: finishing rollback tid 0 to smithi11075562-50 2024-09-14T03:47:49.796 INFO:tasks.rados.rados.0.smithi110.stdout:2920: finishing rollback tid 1 to smithi11075562-50 2024-09-14T03:47:49.796 INFO:tasks.rados.rados.0.smithi110.stdout:2920: finishing rollback tid 2 to smithi11075562-50 2024-09-14T03:47:49.796 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 1505 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:47:49.797 INFO:tasks.rados.rados.0.smithi110.stdout:2918: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:47:49.805 INFO:tasks.rados.rados.0.smithi110.stdout:2918: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:47:49.805 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2030 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-14T03:47:49.806 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1843 (ObjNum 820 snap 272 seq_num 820) dirty exists 2024-09-14T03:47:49.817 INFO:tasks.rados.rados.0.smithi110.stdout:2919: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:47:49.819 INFO:tasks.rados.rados.0.smithi110.stdout:2919: finishing copy_from to smithi11075562-49 2024-09-14T03:47:49.820 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1986 (ObjNum 891 snap 289 seq_num 891) dirty exists 2024-09-14T03:47:49.843 INFO:tasks.rados.rados.0.smithi110.stdout:2917: expect (ObjNum 820 snap 272 seq_num 820) 2024-09-14T03:47:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:49 smithi110 ceph-mon[88262]: pgmap v627: 121 pgs: 121 active+clean; 311 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-14T03:47:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:49 smithi110 ceph-mon[88262]: osdmap e667: 8 total, 8 up, 8 in 2024-09-14T03:47:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:49 smithi110 ceph-mon[86423]: pgmap v627: 121 pgs: 121 active+clean; 311 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-14T03:47:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:49 smithi110 ceph-mon[86423]: osdmap e667: 8 total, 8 up, 8 in 2024-09-14T03:47:50.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:49 smithi195 ceph-mon[82388]: pgmap v627: 121 pgs: 121 active+clean; 311 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-14T03:47:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:49 smithi195 ceph-mon[82388]: osdmap e667: 8 total, 8 up, 8 in 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2921: finishing copy_from racing read to smithi11075562-20 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2921: finishing copy_from to smithi11075562-20 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1824 (ObjNum 828 snap 274 seq_num 828) dirty exists 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2916: done (6 left) 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2917: done (5 left) 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2918: done (4 left) 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2919: done (3 left) 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2920: done (2 left) 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2921: done (1 left) 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:2922: done (0 left) 2024-09-14T03:47:50.440 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 297 2024-09-14T03:47:50.441 INFO:tasks.rados.rados.0.smithi110.stdout:2923: seq_num 905 ranges {0=761856} 2024-09-14T03:47:50.443 INFO:tasks.rados.rados.0.smithi110.stdout:2923: writing smithi11075562-15 from 0 to 761856 tid 1 2024-09-14T03:47:50.443 INFO:tasks.rados.rados.0.smithi110.stdout:2924: snap_remove snap 290 2024-09-14T03:47:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:47:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:47:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:47:50.822 INFO:tasks.rados.rados.0.smithi110.stdout:2923: finishing write tid 1 to smithi11075562-15 2024-09-14T03:47:50.823 INFO:tasks.rados.rados.0.smithi110.stdout:2923: finishing write tid 2 to smithi11075562-15 2024-09-14T03:47:50.823 INFO:tasks.rados.rados.0.smithi110.stdout:2923: finishing write tid 3 to smithi11075562-15 2024-09-14T03:47:50.823 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1824 (ObjNum 905 snap 297 seq_num 905) dirty exists 2024-09-14T03:47:50.823 INFO:tasks.rados.rados.0.smithi110.stdout:2923: left oid 15 (ObjNum 905 snap 297 seq_num 905) 2024-09-14T03:47:50.823 INFO:tasks.rados.rados.0.smithi110.stdout:2923: done (1 left) 2024-09-14T03:47:50.823 INFO:tasks.rados.rados.0.smithi110.stdout:2924: done (0 left) 2024-09-14T03:47:50.823 INFO:tasks.rados.rados.0.smithi110.stdout:2925: delete oid 20 current snap is 297 2024-09-14T03:47:50.829 INFO:tasks.rados.rados.0.smithi110.stdout:2925: done (0 left) 2024-09-14T03:47:50.829 INFO:tasks.rados.rados.0.smithi110.stdout:2926: snap_remove snap 295 2024-09-14T03:47:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:50 smithi110 ceph-mon[88262]: pgmap v630: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-14T03:47:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:50 smithi110 ceph-mon[86423]: pgmap v630: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-14T03:47:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:50 smithi195 ceph-mon[82388]: pgmap v630: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-14T03:47:51.824 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:47:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:47:51] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-14T03:47:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:2926: done (0 left) 2024-09-14T03:47:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:2927: read oid 45 snap 293 2024-09-14T03:47:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:2927: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:47:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:2928: setattr oid 8 current snap is 297 2024-09-14T03:47:51.833 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 297 2024-09-14T03:47:51.833 INFO:tasks.rados.rados.0.smithi110.stdout:2929: seq_num 907 ranges {671744=794624} 2024-09-14T03:47:51.843 INFO:tasks.rados.rados.0.smithi110.stdout:2929: writing smithi11075562-49 from 671744 to 1466368 tid 1 2024-09-14T03:47:51.843 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 297 2024-09-14T03:47:51.843 INFO:tasks.rados.rados.0.smithi110.stdout:2930: seq_num 908 ranges {0=409600,409600=401408} 2024-09-14T03:47:51.845 INFO:tasks.rados.rados.0.smithi110.stdout:2930: writing smithi11075562-17 from 0 to 409600 tid 1 2024-09-14T03:47:51.848 INFO:tasks.rados.rados.0.smithi110.stdout:2930: writing smithi11075562-17 from 409600 to 811008 tid 2 2024-09-14T03:47:51.848 INFO:tasks.rados.rados.0.smithi110.stdout:2931: snap_remove snap 273 2024-09-14T03:47:52.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:51 smithi110 ceph-mon[88262]: osdmap e668: 8 total, 8 up, 8 in 2024-09-14T03:47:52.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:51 smithi110 ceph-mon[86423]: osdmap e668: 8 total, 8 up, 8 in 2024-09-14T03:47:52.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:51 smithi195 ceph-mon[82388]: osdmap e668: 8 total, 8 up, 8 in 2024-09-14T03:47:52.837 INFO:tasks.rados.rados.0.smithi110.stdout:2931: done (4 left) 2024-09-14T03:47:52.837 INFO:tasks.rados.rados.0.smithi110.stdout:2932: delete oid 38 current snap is 297 2024-09-14T03:47:52.837 INFO:tasks.rados.rados.0.smithi110.stdout:2927: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:47:52.861 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1825 (ObjNum 896 snap 292 seq_num 896) dirty exists 2024-09-14T03:47:52.861 INFO:tasks.rados.rados.0.smithi110.stdout:2929: finishing write tid 1 to smithi11075562-49 2024-09-14T03:47:52.861 INFO:tasks.rados.rados.0.smithi110.stdout:2929: finishing write tid 2 to smithi11075562-49 2024-09-14T03:47:52.861 INFO:tasks.rados.rados.0.smithi110.stdout:2930: finishing write tid 1 to smithi11075562-17 2024-09-14T03:47:52.861 INFO:tasks.rados.rados.0.smithi110.stdout:2929: finishing write tid 3 to smithi11075562-49 2024-09-14T03:47:52.861 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1988 (ObjNum 907 snap 297 seq_num 907) dirty exists 2024-09-14T03:47:52.861 INFO:tasks.rados.rados.0.smithi110.stdout:2929: left oid 49 (ObjNum 907 snap 297 seq_num 907) 2024-09-14T03:47:52.861 INFO:tasks.rados.rados.0.smithi110.stdout:2930: finishing write tid 2 to smithi11075562-17 2024-09-14T03:47:52.862 INFO:tasks.rados.rados.0.smithi110.stdout:2930: finishing write tid 3 to smithi11075562-17 2024-09-14T03:47:52.862 INFO:tasks.rados.rados.0.smithi110.stdout:2930: finishing write tid 4 to smithi11075562-17 2024-09-14T03:47:52.862 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1590 (ObjNum 908 snap 297 seq_num 908) dirty exists 2024-09-14T03:47:52.862 INFO:tasks.rados.rados.0.smithi110.stdout:2930: left oid 17 (ObjNum 908 snap 297 seq_num 908) 2024-09-14T03:47:52.863 INFO:tasks.rados.rados.0.smithi110.stdout:2927: done (4 left) 2024-09-14T03:47:52.863 INFO:tasks.rados.rados.0.smithi110.stdout:2928: done (3 left) 2024-09-14T03:47:52.863 INFO:tasks.rados.rados.0.smithi110.stdout:2929: done (2 left) 2024-09-14T03:47:52.863 INFO:tasks.rados.rados.0.smithi110.stdout:2930: done (1 left) 2024-09-14T03:47:52.863 INFO:tasks.rados.rados.0.smithi110.stdout:2932: done (0 left) 2024-09-14T03:47:52.863 INFO:tasks.rados.rados.0.smithi110.stdout:2933: rollback oid 31 current snap is 297 2024-09-14T03:47:52.863 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 296 2024-09-14T03:47:52.864 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 297 2024-09-14T03:47:52.864 INFO:tasks.rados.rados.0.smithi110.stdout:2934: seq_num 910 ranges {0=442368,442368=401408} 2024-09-14T03:47:52.865 INFO:tasks.rados.rados.0.smithi110.stdout:2934: writing smithi11075562-29 from 0 to 442368 tid 1 2024-09-14T03:47:52.868 INFO:tasks.rados.rados.0.smithi110.stdout:2934: writing smithi11075562-29 from 442368 to 843776 tid 2 2024-09-14T03:47:52.868 INFO:tasks.rados.rados.0.smithi110.stdout:2935: read oid 48 snap 293 2024-09-14T03:47:52.868 INFO:tasks.rados.rados.0.smithi110.stdout:2935: expect deleted 2024-09-14T03:47:52.868 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 297 2024-09-14T03:47:52.868 INFO:tasks.rados.rados.0.smithi110.stdout:2936: seq_num 911 ranges {671744=466944,1138688=401408} 2024-09-14T03:47:52.874 INFO:tasks.rados.rados.0.smithi110.stdout:2936: writing smithi11075562-12 from 671744 to 1138688 tid 1 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2936: writing smithi11075562-12 from 1138688 to 1540096 tid 2 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2937: copy_from oid 24 from oid 39 current snap is 297 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2933: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 0 (ObjNum 129 snap 0 seq_num 66708600) dirty dne 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2933: done (4 left) 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2938: delete oid 11 current snap is 297 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2934: finishing write tid 1 to smithi11075562-29 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2934: finishing write tid 2 to smithi11075562-29 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2934: finishing write tid 3 to smithi11075562-29 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2934: finishing write tid 4 to smithi11075562-29 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1615 (ObjNum 910 snap 297 seq_num 910) dirty exists 2024-09-14T03:47:52.877 INFO:tasks.rados.rados.0.smithi110.stdout:2934: left oid 29 (ObjNum 910 snap 297 seq_num 910) 2024-09-14T03:47:52.880 INFO:tasks.rados.rados.0.smithi110.stdout:2934: done (4 left) 2024-09-14T03:47:52.880 INFO:tasks.rados.rados.0.smithi110.stdout:2935: done (3 left) 2024-09-14T03:47:52.880 INFO:tasks.rados.rados.0.smithi110.stdout:2938: done (2 left) 2024-09-14T03:47:52.880 INFO:tasks.rados.rados.0.smithi110.stdout:2939: read oid 5 snap -1 2024-09-14T03:47:52.880 INFO:tasks.rados.rados.0.smithi110.stdout:2939: expect deleted 2024-09-14T03:47:52.880 INFO:tasks.rados.rados.0.smithi110.stdout:2940: copy_from oid 44 from oid 30 current snap is 297 2024-09-14T03:47:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 20 current snap is 297 2024-09-14T03:47:52.881 INFO:tasks.rados.rados.0.smithi110.stdout:2941: seq_num 913 ranges {0=499712,499712=401408} 2024-09-14T03:47:52.883 INFO:tasks.rados.rados.0.smithi110.stdout:2941: writing smithi11075562-20 from 0 to 499712 tid 1 2024-09-14T03:47:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:2941: writing smithi11075562-20 from 499712 to 901120 tid 2 2024-09-14T03:47:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:2942: delete oid 21 current snap is 297 2024-09-14T03:47:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:2936: finishing write tid 1 to smithi11075562-12 2024-09-14T03:47:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:2936: finishing write tid 2 to smithi11075562-12 2024-09-14T03:47:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:2936: finishing write tid 3 to smithi11075562-12 2024-09-14T03:47:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:2936: finishing write tid 4 to smithi11075562-12 2024-09-14T03:47:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2048 (ObjNum 911 snap 297 seq_num 911) dirty exists 2024-09-14T03:47:52.886 INFO:tasks.rados.rados.0.smithi110.stdout:2936: left oid 12 (ObjNum 911 snap 297 seq_num 911) 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:2936: done (5 left) 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:2939: done (4 left) 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:2942: done (3 left) 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:2943: rollback oid 17 current snap is 297 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 17 to 296 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:2944: copy_from oid 26 from oid 25 current snap is 297 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:2945: rollback oid 42 current snap is 297 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 279 2024-09-14T03:47:52.890 INFO:tasks.rados.rados.0.smithi110.stdout:2946: delete oid 47 current snap is 297 2024-09-14T03:47:52.891 INFO:tasks.rados.rados.0.smithi110.stdout:2941: finishing write tid 1 to smithi11075562-20 2024-09-14T03:47:52.891 INFO:tasks.rados.rados.0.smithi110.stdout:2941: finishing write tid 2 to smithi11075562-20 2024-09-14T03:47:52.891 INFO:tasks.rados.rados.0.smithi110.stdout:2941: finishing write tid 3 to smithi11075562-20 2024-09-14T03:47:52.892 INFO:tasks.rados.rados.0.smithi110.stdout:2941: finishing write tid 4 to smithi11075562-20 2024-09-14T03:47:52.893 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1831 (ObjNum 913 snap 297 seq_num 913) dirty exists 2024-09-14T03:47:52.893 INFO:tasks.rados.rados.0.smithi110.stdout:2941: left oid 20 (ObjNum 913 snap 297 seq_num 913) 2024-09-14T03:47:52.893 INFO:tasks.rados.rados.0.smithi110.stdout:2943: finishing rollback tid 0 to smithi11075562-17 2024-09-14T03:47:52.893 INFO:tasks.rados.rados.0.smithi110.stdout:2944: finishing copy_from to smithi11075562-26 2024-09-14T03:47:52.894 INFO:tasks.rados.rados.0.smithi110.stdout:2944: got expected ENOENT (src dne) 2024-09-14T03:47:52.894 INFO:tasks.rados.rados.0.smithi110.stdout:2944: finishing copy_from racing read to smithi11075562-26 2024-09-14T03:47:52.894 INFO:tasks.rados.rados.0.smithi110.stdout:2943: finishing rollback tid 1 to smithi11075562-17 2024-09-14T03:47:52.894 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1592 (ObjNum 257 snap 0 seq_num 2917113296) dirty dne 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2941: done (6 left) 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2943: done (5 left) 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2944: done (4 left) 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2946: done (3 left) 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2947: read oid 22 snap -1 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2947: expect deleted 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2948: read oid 49 snap -1 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2948: expect (ObjNum 907 snap 297 seq_num 907) 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2949: read oid 21 snap -1 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2949: expect deleted 2024-09-14T03:47:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:2950: snap_create 2024-09-14T03:47:52.896 INFO:tasks.rados.rados.0.smithi110.stdout:2945: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:47:52.896 INFO:tasks.rados.rados.0.smithi110.stdout:2945: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:47:52.902 INFO:tasks.rados.rados.0.smithi110.stdout:2945: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:47:52.902 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1991 (ObjNum 842 snap 275 seq_num 842) dirty exists 2024-09-14T03:47:52.904 INFO:tasks.rados.rados.0.smithi110.stdout:2948: expect (ObjNum 907 snap 297 seq_num 907) 2024-09-14T03:47:52.964 INFO:tasks.rados.rados.0.smithi110.stdout:2940: finishing copy_from racing read to smithi11075562-44 2024-09-14T03:47:52.964 INFO:tasks.rados.rados.0.smithi110.stdout:2940: finishing copy_from to smithi11075562-44 2024-09-14T03:47:52.964 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 1808 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:47:52.964 INFO:tasks.rados.rados.0.smithi110.stdout:2937: finishing copy_from racing read to smithi11075562-24 2024-09-14T03:47:52.964 INFO:tasks.rados.rados.0.smithi110.stdout:2937: finishing copy_from to smithi11075562-24 2024-09-14T03:47:52.965 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1514 (ObjNum 789 snap 259 seq_num 789) dirty exists 2024-09-14T03:47:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:52 smithi110 ceph-mon[88262]: pgmap v632: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 268 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 337 KiB/s wr, 10 op/s 2024-09-14T03:47:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:52 smithi110 ceph-mon[88262]: osdmap e669: 8 total, 8 up, 8 in 2024-09-14T03:47:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:52 smithi110 ceph-mon[86423]: pgmap v632: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 268 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 337 KiB/s wr, 10 op/s 2024-09-14T03:47:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:52 smithi110 ceph-mon[86423]: osdmap e669: 8 total, 8 up, 8 in 2024-09-14T03:47:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:52 smithi195 ceph-mon[82388]: pgmap v632: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 268 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 337 KiB/s wr, 10 op/s 2024-09-14T03:47:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:52 smithi195 ceph-mon[82388]: osdmap e669: 8 total, 8 up, 8 in 2024-09-14T03:47:53.778 INFO:tasks.rados.rados.0.smithi110.stdout:2937: done (6 left) 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2940: done (5 left) 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2945: done (4 left) 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2947: done (3 left) 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2948: done (2 left) 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2949: done (1 left) 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2950: done (0 left) 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2951: rollback oid 3 current snap is 298 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 3 to 291 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2952: read oid 39 snap -1 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2952: expect (ObjNum 789 snap 259 seq_num 789) 2024-09-14T03:47:53.779 INFO:tasks.rados.rados.0.smithi110.stdout:2953: snap_create 2024-09-14T03:47:53.787 INFO:tasks.rados.rados.0.smithi110.stdout:2951: finishing rollback tid 0 to smithi11075562-3 2024-09-14T03:47:53.788 INFO:tasks.rados.rados.0.smithi110.stdout:2951: finishing rollback tid 1 to smithi11075562-3 2024-09-14T03:47:53.788 INFO:tasks.rados.rados.0.smithi110.stdout:2951: finishing rollback tid 2 to smithi11075562-3 2024-09-14T03:47:53.788 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 1665 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:47:53.832 INFO:tasks.rados.rados.0.smithi110.stdout:2952: expect (ObjNum 789 snap 259 seq_num 789) 2024-09-14T03:47:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:53 smithi110 ceph-mon[88262]: osdmap e670: 8 total, 8 up, 8 in 2024-09-14T03:47:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:53 smithi110 ceph-mon[88262]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T03:47:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:53 smithi110 ceph-mon[86423]: osdmap e670: 8 total, 8 up, 8 in 2024-09-14T03:47:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:53 smithi110 ceph-mon[86423]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T03:47:54.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:53 smithi195 ceph-mon[82388]: osdmap e670: 8 total, 8 up, 8 in 2024-09-14T03:47:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:53 smithi195 ceph-mon[82388]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T03:47:55.510 INFO:tasks.rados.rados.0.smithi110.stdout:2951: done (2 left) 2024-09-14T03:47:55.510 INFO:tasks.rados.rados.0.smithi110.stdout:2952: done (1 left) 2024-09-14T03:47:55.510 INFO:tasks.rados.rados.0.smithi110.stdout:2953: done (0 left) 2024-09-14T03:47:55.511 INFO:tasks.rados.rados.0.smithi110.stdout:2954: delete oid 12 current snap is 299 2024-09-14T03:47:55.513 INFO:tasks.rados.rados.0.smithi110.stdout:2954: done (0 left) 2024-09-14T03:47:55.514 INFO:tasks.rados.rados.0.smithi110.stdout:2955: read oid 38 snap -1 2024-09-14T03:47:55.514 INFO:tasks.rados.rados.0.smithi110.stdout:2955: expect deleted 2024-09-14T03:47:55.514 INFO:tasks.rados.rados.0.smithi110.stdout:2956: read oid 47 snap 279 2024-09-14T03:47:55.514 INFO:tasks.rados.rados.0.smithi110.stdout:2956: expect (ObjNum 838 snap 275 seq_num 838) 2024-09-14T03:47:55.514 INFO:tasks.rados.rados.0.smithi110.stdout:2957: delete oid 26 current snap is 299 2024-09-14T03:47:55.524 INFO:tasks.rados.rados.0.smithi110.stdout:2956: expect (ObjNum 838 snap 275 seq_num 838) 2024-09-14T03:47:55.576 INFO:tasks.rados.rados.0.smithi110.stdout:2955: done (2 left) 2024-09-14T03:47:55.576 INFO:tasks.rados.rados.0.smithi110.stdout:2956: done (1 left) 2024-09-14T03:47:55.576 INFO:tasks.rados.rados.0.smithi110.stdout:2957: done (0 left) 2024-09-14T03:47:55.576 INFO:tasks.rados.rados.0.smithi110.stdout:2958: snap_remove snap 291 2024-09-14T03:47:55.788 INFO:tasks.rados.rados.0.smithi110.stdout:2958: done (0 left) 2024-09-14T03:47:55.789 INFO:tasks.rados.rados.0.smithi110.stdout:2959: snap_remove snap 296 2024-09-14T03:47:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:55 smithi110 ceph-mon[88262]: pgmap v636: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 255 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-14T03:47:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:55 smithi110 ceph-mon[88262]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T03:47:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:55 smithi110 ceph-mon[86423]: pgmap v636: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 255 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-14T03:47:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:55 smithi110 ceph-mon[86423]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T03:47:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:55 smithi195 ceph-mon[82388]: pgmap v636: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 255 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-14T03:47:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:55 smithi195 ceph-mon[82388]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2959: done (0 left) 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2960: rollback oid 12 current snap is 299 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 294 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2961: rmattr oid 23 current snap is 299 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2961: done (1 left) 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2962: read oid 2 snap -1 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2962: expect (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2963: read oid 34 snap -1 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2963: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:56.794 INFO:tasks.rados.rados.0.smithi110.stdout:2964: setattr oid 30 current snap is 299 2024-09-14T03:47:56.796 INFO:tasks.rados.rados.0.smithi110.stdout:2965: rollback oid 9 current snap is 299 2024-09-14T03:47:56.796 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 9 to 298 2024-09-14T03:47:56.796 INFO:tasks.rados.rados.0.smithi110.stdout:2966: rmattr oid 44 current snap is 299 2024-09-14T03:47:56.796 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 299 2024-09-14T03:47:56.796 INFO:tasks.rados.rados.0.smithi110.stdout:2967: seq_num 918 ranges {0=581632,581632=401408} 2024-09-14T03:47:56.799 INFO:tasks.rados.rados.0.smithi110.stdout:2967: writing smithi11075562-11 from 0 to 581632 tid 1 2024-09-14T03:47:56.802 INFO:tasks.rados.rados.0.smithi110.stdout:2967: writing smithi11075562-11 from 581632 to 983040 tid 2 2024-09-14T03:47:56.803 INFO:tasks.rados.rados.0.smithi110.stdout:2965: finishing rollback tid 1 to smithi11075562-9 2024-09-14T03:47:56.803 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-14T03:47:56.803 INFO:tasks.rados.rados.0.smithi110.stdout:2963: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:47:56.805 INFO:tasks.rados.rados.0.smithi110.stdout:2962: expect (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:47:56.817 INFO:tasks.rados.rados.0.smithi110.stdout:2960: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:47:56.817 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 1809 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:47:56.817 INFO:tasks.rados.rados.0.smithi110.stdout:2960: finishing rollback tid 2 to smithi11075562-12 2024-09-14T03:47:56.817 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2060 (ObjNum 891 snap 289 seq_num 891) dirty exists 2024-09-14T03:47:56.817 INFO:tasks.rados.rados.0.smithi110.stdout:2967: finishing write tid 1 to smithi11075562-11 2024-09-14T03:47:56.817 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1826 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:47:56.817 INFO:tasks.rados.rados.0.smithi110.stdout:2967: finishing write tid 2 to smithi11075562-11 2024-09-14T03:47:56.818 INFO:tasks.rados.rados.0.smithi110.stdout:2960: done (6 left) 2024-09-14T03:47:56.818 INFO:tasks.rados.rados.0.smithi110.stdout:2962: done (5 left) 2024-09-14T03:47:56.818 INFO:tasks.rados.rados.0.smithi110.stdout:2963: done (4 left) 2024-09-14T03:47:56.818 INFO:tasks.rados.rados.0.smithi110.stdout:2964: done (3 left) 2024-09-14T03:47:56.818 INFO:tasks.rados.rados.0.smithi110.stdout:2965: done (2 left) 2024-09-14T03:47:56.818 INFO:tasks.rados.rados.0.smithi110.stdout:2966: done (1 left) 2024-09-14T03:47:56.818 INFO:tasks.rados.rados.0.smithi110.stdout:2968: delete oid 26 current snap is 299 2024-09-14T03:47:56.818 INFO:tasks.rados.rados.0.smithi110.stdout:2967: finishing write tid 3 to smithi11075562-11 2024-09-14T03:47:56.819 INFO:tasks.rados.rados.0.smithi110.stdout:2967: finishing write tid 4 to smithi11075562-11 2024-09-14T03:47:56.819 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1595 (ObjNum 918 snap 299 seq_num 918) dirty exists 2024-09-14T03:47:56.819 INFO:tasks.rados.rados.0.smithi110.stdout:2967: left oid 11 (ObjNum 918 snap 299 seq_num 918) 2024-09-14T03:47:56.820 INFO:tasks.rados.rados.0.smithi110.stdout:2967: done (1 left) 2024-09-14T03:47:56.820 INFO:tasks.rados.rados.0.smithi110.stdout:2968: done (0 left) 2024-09-14T03:47:56.820 INFO:tasks.rados.rados.0.smithi110.stdout:2969: copy_from oid 34 from oid 29 current snap is 299 2024-09-14T03:47:56.820 INFO:tasks.rados.rados.0.smithi110.stdout:2970: delete oid 14 current snap is 299 2024-09-14T03:47:56.821 INFO:tasks.rados.rados.0.smithi110.stdout:2970: done (1 left) 2024-09-14T03:47:56.821 INFO:tasks.rados.rados.0.smithi110.stdout:2971: setattr oid 36 current snap is 299 2024-09-14T03:47:56.824 INFO:tasks.rados.rados.0.smithi110.stdout:2972: rollback oid 10 current snap is 299 2024-09-14T03:47:56.824 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 297 2024-09-14T03:47:56.824 INFO:tasks.rados.rados.0.smithi110.stdout:2973: copy_from oid 46 from oid 49 current snap is 299 2024-09-14T03:47:56.825 INFO:tasks.rados.rados.0.smithi110.stdout:2974: rollback oid 24 current snap is 299 2024-09-14T03:47:56.825 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 292 2024-09-14T03:47:56.825 INFO:tasks.rados.rados.0.smithi110.stdout:2975: read oid 15 snap -1 2024-09-14T03:47:56.825 INFO:tasks.rados.rados.0.smithi110.stdout:2975: expect (ObjNum 905 snap 297 seq_num 905) 2024-09-14T03:47:56.825 INFO:tasks.rados.rados.0.smithi110.stdout:2976: rollback oid 38 current snap is 299 2024-09-14T03:47:56.825 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 297 2024-09-14T03:47:56.825 INFO:tasks.rados.rados.0.smithi110.stdout:2977: setattr oid 21 current snap is 299 2024-09-14T03:47:56.825 INFO:tasks.rados.rados.0.smithi110.stdout:2969: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:47:56.826 INFO:tasks.rados.rados.0.smithi110.stdout:2976: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:47:56.827 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1612 (ObjNum 97 snap 0 seq_num 65128912) dirty dne 2024-09-14T03:47:56.827 INFO:tasks.rados.rados.0.smithi110.stdout:2969: finishing copy_from to smithi11075562-34 2024-09-14T03:47:56.827 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1615 (ObjNum 910 snap 297 seq_num 910) dirty exists 2024-09-14T03:47:56.827 INFO:tasks.rados.rados.0.smithi110.stdout:2969: done (7 left) 2024-09-14T03:47:56.827 INFO:tasks.rados.rados.0.smithi110.stdout:2976: done (6 left) 2024-09-14T03:47:56.828 INFO:tasks.rados.rados.0.smithi110.stdout:2978: rmattr oid 3 current snap is 299 2024-09-14T03:47:56.828 INFO:tasks.rados.rados.0.smithi110.stdout:2979: snap_remove snap 294 2024-09-14T03:47:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:56 smithi110 ceph-mon[88262]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T03:47:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:56 smithi110 ceph-mon[86423]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T03:47:57.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:56 smithi195 ceph-mon[82388]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T03:47:57.800 INFO:tasks.rados.rados.0.smithi110.stdout:2979: done (7 left) 2024-09-14T03:47:57.800 INFO:tasks.rados.rados.0.smithi110.stdout:2980: rollback oid 29 current snap is 299 2024-09-14T03:47:57.800 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 29 to 292 2024-09-14T03:47:57.800 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 299 2024-09-14T03:47:57.801 INFO:tasks.rados.rados.0.smithi110.stdout:2981: seq_num 921 ranges {0=630784,630784=401408} 2024-09-14T03:47:57.804 INFO:tasks.rados.rados.0.smithi110.stdout:2981: writing smithi11075562-16 from 0 to 630784 tid 1 2024-09-14T03:47:57.807 INFO:tasks.rados.rados.0.smithi110.stdout:2981: writing smithi11075562-16 from 630784 to 1032192 tid 2 2024-09-14T03:47:57.807 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2068 (ObjNum 129 snap 0 seq_num 64494920) dirty exists 2024-09-14T03:47:57.807 INFO:tasks.rados.rados.0.smithi110.stdout:2975: expect (ObjNum 905 snap 297 seq_num 905) 2024-09-14T03:47:57.834 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 1666 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:47:57.834 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 1827 (ObjNum 65 snap 0 seq_num 1494375228) dirty exists 2024-09-14T03:47:57.834 INFO:tasks.rados.rados.0.smithi110.stdout:2972: finishing rollback tid 0 to smithi11075562-10 2024-09-14T03:47:57.834 INFO:tasks.rados.rados.0.smithi110.stdout:2972: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2972: finishing rollback tid 2 to smithi11075562-10 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1885 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2974: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2974: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2973: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2974: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1888 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2973: finishing copy_from to smithi11075562-46 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1988 (ObjNum 907 snap 297 seq_num 907) dirty exists 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2971: done (8 left) 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2972: done (7 left) 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2973: done (6 left) 2024-09-14T03:47:57.835 INFO:tasks.rados.rados.0.smithi110.stdout:2974: done (5 left) 2024-09-14T03:47:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2975: done (4 left) 2024-09-14T03:47:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2977: done (3 left) 2024-09-14T03:47:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2978: done (2 left) 2024-09-14T03:47:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 299 2024-09-14T03:47:57.836 INFO:tasks.rados.rados.0.smithi110.stdout:2982: seq_num 922 ranges {614400=647168,1261568=401408} 2024-09-14T03:47:57.841 INFO:tasks.rados.rados.0.smithi110.stdout:2982: writing smithi11075562-8 from 614400 to 1261568 tid 1 2024-09-14T03:47:57.844 INFO:tasks.rados.rados.0.smithi110.stdout:2982: writing smithi11075562-8 from 1261568 to 1662976 tid 2 2024-09-14T03:47:57.845 INFO:tasks.rados.rados.0.smithi110.stdout:2983: rollback oid 39 current snap is 299 2024-09-14T03:47:57.845 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 298 2024-09-14T03:47:57.845 INFO:tasks.rados.rados.0.smithi110.stdout:2980: finishing rollback tid 0 to smithi11075562-29 2024-09-14T03:47:57.845 INFO:tasks.rados.rados.0.smithi110.stdout:2980: finishing rollback tid 1 to smithi11075562-29 2024-09-14T03:47:57.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1617 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-14T03:47:57.845 INFO:tasks.rados.rados.0.smithi110.stdout:2981: finishing write tid 1 to smithi11075562-16 2024-09-14T03:47:57.845 INFO:tasks.rados.rados.0.smithi110.stdout:2981: finishing write tid 2 to smithi11075562-16 2024-09-14T03:47:57.845 INFO:tasks.rados.rados.0.smithi110.stdout:2981: finishing write tid 3 to smithi11075562-16 2024-09-14T03:47:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:2981: finishing write tid 4 to smithi11075562-16 2024-09-14T03:47:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 1554 (ObjNum 921 snap 299 seq_num 921) dirty exists 2024-09-14T03:47:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:2981: left oid 16 (ObjNum 921 snap 299 seq_num 921) 2024-09-14T03:47:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:2980: done (3 left) 2024-09-14T03:47:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:2981: done (2 left) 2024-09-14T03:47:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:2984: copy_from oid 31 from oid 35 current snap is 299 2024-09-14T03:47:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 299 2024-09-14T03:47:57.846 INFO:tasks.rados.rados.0.smithi110.stdout:2985: seq_num 923 ranges {761856=663552,1425408=401408} 2024-09-14T03:47:57.852 INFO:tasks.rados.rados.0.smithi110.stdout:2985: writing smithi11075562-15 from 761856 to 1425408 tid 1 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:2985: writing smithi11075562-15 from 1425408 to 1826816 tid 2 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:2986: copy_from oid 5 from oid 41 current snap is 299 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:2984: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:2982: finishing write tid 1 to smithi11075562-8 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:2982: finishing write tid 2 to smithi11075562-8 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:2987: rollback oid 33 current snap is 299 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 33 to 292 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:2984: finishing copy_from to smithi11075562-31 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1838 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:47:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:2982: finishing write tid 3 to smithi11075562-8 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2983: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2984: done (5 left) 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2988: copy_from oid 34 from oid 17 current snap is 299 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2983: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2983: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1830 (ObjNum 789 snap 259 seq_num 789) dirty exists 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2982: finishing write tid 4 to smithi11075562-8 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1829 (ObjNum 922 snap 299 seq_num 922) dirty exists 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2982: left oid 8 (ObjNum 922 snap 299 seq_num 922) 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2982: done (5 left) 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2983: done (4 left) 2024-09-14T03:47:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:2989: snap_create 2024-09-14T03:47:57.858 INFO:tasks.rados.rados.0.smithi110.stdout:2988: finishing copy_from to smithi11075562-34 2024-09-14T03:47:57.858 INFO:tasks.rados.rados.0.smithi110.stdout:2988: got expected ENOENT (src dne) 2024-09-14T03:47:57.858 INFO:tasks.rados.rados.0.smithi110.stdout:2988: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:47:57.858 INFO:tasks.rados.rados.0.smithi110.stdout:2987: finishing rollback tid 1 to smithi11075562-33 2024-09-14T03:47:57.858 INFO:tasks.rados.rados.0.smithi110.stdout:2987: finishing rollback tid 2 to smithi11075562-33 2024-09-14T03:47:57.859 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1990 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:47:57.859 INFO:tasks.rados.rados.0.smithi110.stdout:2985: finishing write tid 1 to smithi11075562-15 2024-09-14T03:47:57.859 INFO:tasks.rados.rados.0.smithi110.stdout:2985: finishing write tid 2 to smithi11075562-15 2024-09-14T03:47:57.859 INFO:tasks.rados.rados.0.smithi110.stdout:2985: finishing write tid 3 to smithi11075562-15 2024-09-14T03:47:57.861 INFO:tasks.rados.rados.0.smithi110.stdout:2985: finishing write tid 4 to smithi11075562-15 2024-09-14T03:47:57.861 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1832 (ObjNum 923 snap 299 seq_num 923) dirty exists 2024-09-14T03:47:57.861 INFO:tasks.rados.rados.0.smithi110.stdout:2985: left oid 15 (ObjNum 923 snap 299 seq_num 923) 2024-09-14T03:47:57.872 INFO:tasks.rados.rados.0.smithi110.stdout:2986: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:47:57.874 INFO:tasks.rados.rados.0.smithi110.stdout:2986: finishing copy_from to smithi11075562-5 2024-09-14T03:47:57.874 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 1824 (ObjNum 828 snap 274 seq_num 828) dirty exists 2024-09-14T03:47:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:57 smithi110 ceph-mon[88262]: pgmap v639: 121 pgs: 121 active+clean; 229 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-14T03:47:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:57 smithi110 ceph-mon[88262]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T03:47:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:57 smithi110 ceph-mon[86423]: pgmap v639: 121 pgs: 121 active+clean; 229 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-14T03:47:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:57 smithi110 ceph-mon[86423]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T03:47:58.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:58.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:57 smithi195 ceph-mon[82388]: pgmap v639: 121 pgs: 121 active+clean; 229 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-14T03:47:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:57 smithi195 ceph-mon[82388]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T03:47:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:47:58.780 INFO:tasks.rados.rados.0.smithi110.stdout:2985: done (4 left) 2024-09-14T03:47:58.780 INFO:tasks.rados.rados.0.smithi110.stdout:2986: done (3 left) 2024-09-14T03:47:58.781 INFO:tasks.rados.rados.0.smithi110.stdout:2987: done (2 left) 2024-09-14T03:47:58.781 INFO:tasks.rados.rados.0.smithi110.stdout:2988: done (1 left) 2024-09-14T03:47:58.781 INFO:tasks.rados.rados.0.smithi110.stdout:2989: done (0 left) 2024-09-14T03:47:58.781 INFO:tasks.rados.rados.0.smithi110.stdout:2990: setattr oid 22 current snap is 300 2024-09-14T03:47:58.782 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 300 2024-09-14T03:47:58.782 INFO:tasks.rados.rados.0.smithi110.stdout:2991: seq_num 924 ranges {0=679936,679936=401408} 2024-09-14T03:47:58.786 INFO:tasks.rados.rados.0.smithi110.stdout:2991: writing smithi11075562-26 from 0 to 679936 tid 1 2024-09-14T03:47:58.790 INFO:tasks.rados.rados.0.smithi110.stdout:2991: writing smithi11075562-26 from 679936 to 1081344 tid 2 2024-09-14T03:47:58.790 INFO:tasks.rados.rados.0.smithi110.stdout:2992: read oid 36 snap 289 2024-09-14T03:47:58.790 INFO:tasks.rados.rados.0.smithi110.stdout:2992: expect (ObjNum 66719904 snap 21934 seq_num 65067360) 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:2993: read oid 28 snap -1 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:2993: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 1833 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:2990: done (3 left) 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:2994: copy_from oid 15 from oid 40 current snap is 300 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:2995: rollback oid 39 current snap is 300 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 39 to 293 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:2996: rollback oid 2 current snap is 300 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 2 to 292 2024-09-14T03:47:58.791 INFO:tasks.rados.rados.0.smithi110.stdout:2997: snap_remove snap 299 2024-09-14T03:47:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:58 smithi110 ceph-mon[86423]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T03:47:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:58 smithi110 ceph-mon[86423]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T03:47:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:58 smithi110 ceph-mon[88262]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T03:47:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:58 smithi110 ceph-mon[88262]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T03:47:59.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:58 smithi195 ceph-mon[82388]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T03:47:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:58 smithi195 ceph-mon[82388]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T03:47:59.782 INFO:tasks.rados.rados.0.smithi110.stdout:2997: done (6 left) 2024-09-14T03:47:59.782 INFO:tasks.rados.rados.0.smithi110.stdout:2998: copy_from oid 17 from oid 10 current snap is 300 2024-09-14T03:47:59.782 INFO:tasks.rados.rados.0.smithi110.stdout:2991: finishing write tid 1 to smithi11075562-26 2024-09-14T03:47:59.782 INFO:tasks.rados.rados.0.smithi110.stdout:2991: finishing write tid 2 to smithi11075562-26 2024-09-14T03:47:59.782 INFO:tasks.rados.rados.0.smithi110.stdout:2992: expect (ObjNum 66719904 snap 21934 seq_num 65067360) 2024-09-14T03:47:59.787 INFO:tasks.rados.rados.0.smithi110.stdout:2991: finishing write tid 3 to smithi11075562-26 2024-09-14T03:47:59.787 INFO:tasks.rados.rados.0.smithi110.stdout:2994: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:47:59.787 INFO:tasks.rados.rados.0.smithi110.stdout:2991: finishing write tid 4 to smithi11075562-26 2024-09-14T03:47:59.787 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1829 (ObjNum 924 snap 300 seq_num 924) dirty exists 2024-09-14T03:47:59.788 INFO:tasks.rados.rados.0.smithi110.stdout:2991: left oid 26 (ObjNum 924 snap 300 seq_num 924) 2024-09-14T03:47:59.788 INFO:tasks.rados.rados.0.smithi110.stdout:2995: finishing rollback tid 0 to smithi11075562-39 2024-09-14T03:47:59.788 INFO:tasks.rados.rados.0.smithi110.stdout:2994: finishing copy_from to smithi11075562-15 2024-09-14T03:47:59.788 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1520 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-14T03:47:59.788 INFO:tasks.rados.rados.0.smithi110.stdout:2993: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:47:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:2996: finishing rollback tid 0 to smithi11075562-2 2024-09-14T03:47:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:2996: finishing rollback tid 1 to smithi11075562-2 2024-09-14T03:47:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:2995: finishing rollback tid 1 to smithi11075562-39 2024-09-14T03:47:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:2991: done (6 left) 2024-09-14T03:47:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:2992: done (5 left) 2024-09-14T03:47:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:2993: done (4 left) 2024-09-14T03:47:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:2994: done (3 left) 2024-09-14T03:47:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:2999: snap_remove snap 298 2024-09-14T03:48:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:59 smithi110 ceph-mon[88262]: pgmap v642: 121 pgs: 121 active+clean; 219 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 22 op/s 2024-09-14T03:48:00.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:47:59 smithi110 ceph-mon[88262]: osdmap e677: 8 total, 8 up, 8 in 2024-09-14T03:48:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:59 smithi110 ceph-mon[86423]: pgmap v642: 121 pgs: 121 active+clean; 219 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 22 op/s 2024-09-14T03:48:00.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:47:59 smithi110 ceph-mon[86423]: osdmap e677: 8 total, 8 up, 8 in 2024-09-14T03:48:00.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:59 smithi195 ceph-mon[82388]: pgmap v642: 121 pgs: 121 active+clean; 219 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 22 op/s 2024-09-14T03:48:00.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:47:59 smithi195 ceph-mon[82388]: osdmap e677: 8 total, 8 up, 8 in 2024-09-14T03:48:00.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:48:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:48:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:48:00.812 INFO:tasks.rados.rados.0.smithi110.stdout:2999: done (3 left) 2024-09-14T03:48:00.813 INFO:tasks.rados.rados.0.smithi110.stdout:3000: setattr oid 47 current snap is 300 2024-09-14T03:48:00.813 INFO:tasks.rados.rados.0.smithi110.stdout:2996: finishing rollback tid 2 to smithi11075562-2 2024-09-14T03:48:00.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 1669 (ObjNum 877 snap 281 seq_num 877) dirty exists 2024-09-14T03:48:00.813 INFO:tasks.rados.rados.0.smithi110.stdout:2995: finishing rollback tid 2 to smithi11075562-39 2024-09-14T03:48:00.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1833 (ObjNum 789 snap 259 seq_num 789) dirty exists 2024-09-14T03:48:00.813 INFO:tasks.rados.rados.0.smithi110.stdout:2998: finishing copy_from racing read to smithi11075562-17 2024-09-14T03:48:00.813 INFO:tasks.rados.rados.0.smithi110.stdout:2998: finishing copy_from to smithi11075562-17 2024-09-14T03:48:00.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 1885 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:48:00.814 INFO:tasks.rados.rados.0.smithi110.stdout:2995: done (3 left) 2024-09-14T03:48:00.814 INFO:tasks.rados.rados.0.smithi110.stdout:2996: done (2 left) 2024-09-14T03:48:00.814 INFO:tasks.rados.rados.0.smithi110.stdout:2998: done (1 left) 2024-09-14T03:48:00.814 INFO:tasks.rados.rados.0.smithi110.stdout:3001: copy_from oid 37 from oid 50 current snap is 300 2024-09-14T03:48:00.814 INFO:tasks.rados.rados.0.smithi110.stdout:3002: read oid 1 snap 297 2024-09-14T03:48:00.815 INFO:tasks.rados.rados.0.smithi110.stdout:3002: expect (ObjNum 820 snap 272 seq_num 820) 2024-09-14T03:48:00.815 INFO:tasks.rados.rados.0.smithi110.stdout:3003: snap_create 2024-09-14T03:48:00.826 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 2078 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:48:00.864 INFO:tasks.rados.rados.0.smithi110.stdout:3002: expect (ObjNum 820 snap 272 seq_num 820) 2024-09-14T03:48:01.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:48:01.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:48:01.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:48:01.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:48:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:48:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:48:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:48:01.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:48:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:48:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:48:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:48:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:48:01.465 INFO:tasks.rados.rados.0.smithi110.stdout:3001: finishing copy_from racing read to smithi11075562-37 2024-09-14T03:48:01.465 INFO:tasks.rados.rados.0.smithi110.stdout:3001: finishing copy_from to smithi11075562-37 2024-09-14T03:48:01.465 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1505 (ObjNum 677 snap 211 seq_num 677) dirty exists 2024-09-14T03:48:01.815 INFO:tasks.rados.rados.0.smithi110.stdout:3000: done (3 left) 2024-09-14T03:48:01.816 INFO:tasks.rados.rados.0.smithi110.stdout:3001: done (2 left) 2024-09-14T03:48:01.816 INFO:tasks.rados.rados.0.smithi110.stdout:3002: done (1 left) 2024-09-14T03:48:01.816 INFO:tasks.rados.rados.0.smithi110.stdout:3003: done (0 left) 2024-09-14T03:48:01.816 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 301 2024-09-14T03:48:01.816 INFO:tasks.rados.rados.0.smithi110.stdout:3004: seq_num 925 ranges {983040=696320,1679360=401408} 2024-09-14T03:48:01.824 INFO:tasks.rados.rados.0.smithi110.stdout:3004: writing smithi11075562-11 from 983040 to 1679360 tid 1 2024-09-14T03:48:01.827 INFO:tasks.rados.rados.0.smithi110.stdout:3004: writing smithi11075562-11 from 1679360 to 2080768 tid 2 2024-09-14T03:48:01.827 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 301 2024-09-14T03:48:01.827 INFO:tasks.rados.rados.0.smithi110.stdout:3005: seq_num 926 ranges {0=712704,712704=401408} 2024-09-14T03:48:01.831 INFO:tasks.rados.rados.0.smithi110.stdout:3005: writing smithi11075562-31 from 0 to 712704 tid 1 2024-09-14T03:48:01.833 INFO:tasks.rados.rados.0.smithi110.stdout:3005: writing smithi11075562-31 from 712704 to 1114112 tid 2 2024-09-14T03:48:01.833 INFO:tasks.rados.rados.0.smithi110.stdout:3006: rollback oid 42 current snap is 301 2024-09-14T03:48:01.834 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 283 2024-09-14T03:48:01.834 INFO:tasks.rados.rados.0.smithi110.stdout:3004: finishing write tid 1 to smithi11075562-11 2024-09-14T03:48:01.834 INFO:tasks.rados.rados.0.smithi110.stdout:3004: finishing write tid 2 to smithi11075562-11 2024-09-14T03:48:01.834 INFO:tasks.rados.rados.0.smithi110.stdout:3004: finishing write tid 3 to smithi11075562-11 2024-09-14T03:48:01.834 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 301 2024-09-14T03:48:01.834 INFO:tasks.rados.rados.0.smithi110.stdout:3007: seq_num 927 ranges {1662976=729088,2392064=401408} 2024-09-14T03:48:01.846 INFO:tasks.rados.rados.0.smithi110.stdout:3007: writing smithi11075562-8 from 1662976 to 2392064 tid 1 2024-09-14T03:48:01.849 INFO:tasks.rados.rados.0.smithi110.stdout:3007: writing smithi11075562-8 from 2392064 to 2793472 tid 2 2024-09-14T03:48:01.849 INFO:tasks.rados.rados.0.smithi110.stdout:3008: snap_create 2024-09-14T03:48:01.849 INFO:tasks.rados.rados.0.smithi110.stdout:3004: finishing write tid 4 to smithi11075562-11 2024-09-14T03:48:01.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1888 (ObjNum 925 snap 301 seq_num 925) dirty exists 2024-09-14T03:48:01.849 INFO:tasks.rados.rados.0.smithi110.stdout:3004: left oid 11 (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:3005: finishing write tid 1 to smithi11075562-31 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:3005: finishing write tid 2 to smithi11075562-31 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:3005: finishing write tid 3 to smithi11075562-31 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:3005: finishing write tid 4 to smithi11075562-31 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1841 (ObjNum 926 snap 301 seq_num 926) dirty exists 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:3005: left oid 31 (ObjNum 926 snap 301 seq_num 926) 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:3006: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:3006: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:3006: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:48:01.850 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1994 (ObjNum 879 snap 283 seq_num 879) dirty exists 2024-09-14T03:48:01.852 INFO:tasks.rados.rados.0.smithi110.stdout:3007: finishing write tid 1 to smithi11075562-8 2024-09-14T03:48:01.852 INFO:tasks.rados.rados.0.smithi110.stdout:3007: finishing write tid 2 to smithi11075562-8 2024-09-14T03:48:01.852 INFO:tasks.rados.rados.0.smithi110.stdout:3007: finishing write tid 3 to smithi11075562-8 2024-09-14T03:48:01.853 INFO:tasks.rados.rados.0.smithi110.stdout:3007: finishing write tid 4 to smithi11075562-8 2024-09-14T03:48:01.853 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1836 (ObjNum 927 snap 301 seq_num 927) dirty exists 2024-09-14T03:48:01.854 INFO:tasks.rados.rados.0.smithi110.stdout:3007: left oid 8 (ObjNum 927 snap 301 seq_num 927) 2024-09-14T03:48:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:48:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:48:01] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-14T03:48:01.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:01 smithi110 ceph-mon[88262]: pgmap v645: 121 pgs: 121 active+clean; 238 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-09-14T03:48:01.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:01 smithi110 ceph-mon[88262]: osdmap e678: 8 total, 8 up, 8 in 2024-09-14T03:48:01.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:01 smithi110 ceph-mon[86423]: pgmap v645: 121 pgs: 121 active+clean; 238 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-09-14T03:48:01.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:01 smithi110 ceph-mon[86423]: osdmap e678: 8 total, 8 up, 8 in 2024-09-14T03:48:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:01 smithi195 ceph-mon[82388]: pgmap v645: 121 pgs: 121 active+clean; 238 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-09-14T03:48:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:01 smithi195 ceph-mon[82388]: osdmap e678: 8 total, 8 up, 8 in 2024-09-14T03:48:02.821 INFO:tasks.rados.rados.0.smithi110.stdout:3004: done (4 left) 2024-09-14T03:48:02.821 INFO:tasks.rados.rados.0.smithi110.stdout:3005: done (3 left) 2024-09-14T03:48:02.821 INFO:tasks.rados.rados.0.smithi110.stdout:3006: done (2 left) 2024-09-14T03:48:02.821 INFO:tasks.rados.rados.0.smithi110.stdout:3007: done (1 left) 2024-09-14T03:48:02.822 INFO:tasks.rados.rados.0.smithi110.stdout:3008: done (0 left) 2024-09-14T03:48:02.822 INFO:tasks.rados.rados.0.smithi110.stdout:3009: snap_remove snap 289 2024-09-14T03:48:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:02 smithi110 ceph-mon[88262]: osdmap e679: 8 total, 8 up, 8 in 2024-09-14T03:48:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:02 smithi110 ceph-mon[88262]: pgmap v648: 121 pgs: 121 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-14T03:48:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:02 smithi110 ceph-mon[86423]: osdmap e679: 8 total, 8 up, 8 in 2024-09-14T03:48:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:02 smithi110 ceph-mon[86423]: pgmap v648: 121 pgs: 121 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-14T03:48:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:02 smithi195 ceph-mon[82388]: osdmap e679: 8 total, 8 up, 8 in 2024-09-14T03:48:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:02 smithi195 ceph-mon[82388]: pgmap v648: 121 pgs: 121 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-14T03:48:03.781 INFO:tasks.rados.rados.0.smithi110.stdout:3009: done (0 left) 2024-09-14T03:48:03.781 INFO:tasks.rados.rados.0.smithi110.stdout:3010: setattr oid 40 current snap is 302 2024-09-14T03:48:03.784 INFO:tasks.rados.rados.0.smithi110.stdout:3011: read oid 41 snap 279 2024-09-14T03:48:03.784 INFO:tasks.rados.rados.0.smithi110.stdout:3011: expect (ObjNum 855 snap 278 seq_num 855) 2024-09-14T03:48:03.784 INFO:tasks.rados.rados.0.smithi110.stdout:3012: snap_remove snap 297 2024-09-14T03:48:04.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:03 smithi110 ceph-mon[86423]: osdmap e680: 8 total, 8 up, 8 in 2024-09-14T03:48:04.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:03 smithi110 ceph-mon[86423]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T03:48:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:03 smithi110 ceph-mon[88262]: osdmap e680: 8 total, 8 up, 8 in 2024-09-14T03:48:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:03 smithi110 ceph-mon[88262]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T03:48:04.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:03 smithi195 ceph-mon[82388]: osdmap e680: 8 total, 8 up, 8 in 2024-09-14T03:48:04.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:03 smithi195 ceph-mon[82388]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T03:48:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:3012: done (2 left) 2024-09-14T03:48:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:3013: copy_from oid 27 from oid 44 current snap is 302 2024-09-14T03:48:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 1562 (ObjNum 49 snap 0 seq_num 65148352) dirty exists 2024-09-14T03:48:04.786 INFO:tasks.rados.rados.0.smithi110.stdout:3011: expect (ObjNum 855 snap 278 seq_num 855) 2024-09-14T03:48:04.857 INFO:tasks.rados.rados.0.smithi110.stdout:3013: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:48:04.857 INFO:tasks.rados.rados.0.smithi110.stdout:3013: finishing copy_from to smithi11075562-27 2024-09-14T03:48:04.857 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1809 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:48:04.857 INFO:tasks.rados.rados.0.smithi110.stdout:3010: done (2 left) 2024-09-14T03:48:04.857 INFO:tasks.rados.rados.0.smithi110.stdout:3011: done (1 left) 2024-09-14T03:48:04.857 INFO:tasks.rados.rados.0.smithi110.stdout:3013: done (0 left) 2024-09-14T03:48:04.857 INFO:tasks.rados.rados.0.smithi110.stdout:3014: rmattr oid 1 current snap is 302 2024-09-14T03:48:04.857 INFO:tasks.rados.rados.0.smithi110.stdout:3015: rmattr oid 26 current snap is 302 2024-09-14T03:48:04.858 INFO:tasks.rados.rados.0.smithi110.stdout:3015: done (1 left) 2024-09-14T03:48:04.858 INFO:tasks.rados.rados.0.smithi110.stdout:3016: read oid 48 snap 279 2024-09-14T03:48:04.858 INFO:tasks.rados.rados.0.smithi110.stdout:3016: expect deleted 2024-09-14T03:48:04.858 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 302 2024-09-14T03:48:04.858 INFO:tasks.rados.rados.0.smithi110.stdout:3017: seq_num 928 ranges {0=745472,745472=401408} 2024-09-14T03:48:04.860 INFO:tasks.rados.rados.0.smithi110.stdout:3017: writing smithi11075562-25 from 0 to 745472 tid 1 2024-09-14T03:48:04.863 INFO:tasks.rados.rados.0.smithi110.stdout:3017: writing smithi11075562-25 from 745472 to 1146880 tid 2 2024-09-14T03:48:04.863 INFO:tasks.rados.rados.0.smithi110.stdout:3018: copy_from oid 49 from oid 2 current snap is 302 2024-09-14T03:48:04.863 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1995 (ObjNum 820 snap 272 seq_num 820) dirty exists 2024-09-14T03:48:04.863 INFO:tasks.rados.rados.0.smithi110.stdout:3014: done (3 left) 2024-09-14T03:48:04.863 INFO:tasks.rados.rados.0.smithi110.stdout:3016: done (2 left) 2024-09-14T03:48:04.863 INFO:tasks.rados.rados.0.smithi110.stdout:3019: delete oid 28 current snap is 302 2024-09-14T03:48:04.866 INFO:tasks.rados.rados.0.smithi110.stdout:3019: done (2 left) 2024-09-14T03:48:04.866 INFO:tasks.rados.rados.0.smithi110.stdout:3020: copy_from oid 20 from oid 11 current snap is 302 2024-09-14T03:48:04.866 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 302 2024-09-14T03:48:04.866 INFO:tasks.rados.rados.0.smithi110.stdout:3021: seq_num 930 ranges {704512=786432,1490944=401408} 2024-09-14T03:48:04.874 INFO:tasks.rados.rados.0.smithi110.stdout:3021: writing smithi11075562-13 from 704512 to 1490944 tid 1 2024-09-14T03:48:04.877 INFO:tasks.rados.rados.0.smithi110.stdout:3021: writing smithi11075562-13 from 1490944 to 1892352 tid 2 2024-09-14T03:48:04.877 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 302 2024-09-14T03:48:04.877 INFO:tasks.rados.rados.0.smithi110.stdout:3017: finishing write tid 1 to smithi11075562-25 2024-09-14T03:48:04.877 INFO:tasks.rados.rados.0.smithi110.stdout:3017: finishing write tid 2 to smithi11075562-25 2024-09-14T03:48:04.877 INFO:tasks.rados.rados.0.smithi110.stdout:3017: finishing write tid 3 to smithi11075562-25 2024-09-14T03:48:04.877 INFO:tasks.rados.rados.0.smithi110.stdout:3022: seq_num 931 ranges {9248768=802816,10051584=401408} 2024-09-14T03:48:04.930 INFO:tasks.rados.rados.0.smithi110.stdout:3022: writing smithi11075562-1 from 9248768 to 10051584 tid 1 2024-09-14T03:48:04.933 INFO:tasks.rados.rados.0.smithi110.stdout:3022: writing smithi11075562-1 from 10051584 to 10452992 tid 2 2024-09-14T03:48:04.933 INFO:tasks.rados.rados.0.smithi110.stdout:3023: copy_from oid 48 from oid 10 current snap is 302 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3018: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3018: finishing copy_from to smithi11075562-49 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1669 (ObjNum 877 snap 281 seq_num 877) dirty exists 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3017: finishing write tid 4 to smithi11075562-25 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2103 (ObjNum 928 snap 302 seq_num 928) dirty exists 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3017: left oid 25 (ObjNum 928 snap 302 seq_num 928) 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3020: finishing copy_from racing read to smithi11075562-20 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3017: done (5 left) 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3018: done (4 left) 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3024: rollback oid 30 current snap is 302 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 293 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:3020: finishing copy_from to smithi11075562-20 2024-09-14T03:48:04.934 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 1888 (ObjNum 925 snap 301 seq_num 925) dirty exists 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3021: finishing write tid 1 to smithi11075562-13 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3021: finishing write tid 2 to smithi11075562-13 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3021: finishing write tid 3 to smithi11075562-13 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3021: finishing write tid 4 to smithi11075562-13 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1998 (ObjNum 930 snap 302 seq_num 930) dirty exists 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3021: left oid 13 (ObjNum 930 snap 302 seq_num 930) 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3020: done (4 left) 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3021: done (3 left) 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3025: read oid 47 snap -1 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3025: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:3026: rollback oid 26 current snap is 302 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 293 2024-09-14T03:48:04.935 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 302 2024-09-14T03:48:04.936 INFO:tasks.rados.rados.0.smithi110.stdout:3027: seq_num 932 ranges {4063232=417792,4481024=663552,5144576=139264} 2024-09-14T03:48:04.957 INFO:tasks.rados.rados.0.smithi110.stdout:3027: writing smithi11075562-6 from 4063232 to 4481024 tid 1 2024-09-14T03:48:04.960 INFO:tasks.rados.rados.0.smithi110.stdout:3027: writing smithi11075562-6 from 4481024 to 5144576 tid 2 2024-09-14T03:48:04.961 INFO:tasks.rados.rados.0.smithi110.stdout:3027: writing smithi11075562-6 from 5144576 to 5283840 tid 3 2024-09-14T03:48:04.962 INFO:tasks.rados.rados.0.smithi110.stdout:3028: rollback oid 41 current snap is 302 2024-09-14T03:48:04.962 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 279 2024-09-14T03:48:04.962 INFO:tasks.rados.rados.0.smithi110.stdout:3025: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:04.964 INFO:tasks.rados.rados.0.smithi110.stdout:3022: finishing write tid 1 to smithi11075562-1 2024-09-14T03:48:04.964 INFO:tasks.rados.rados.0.smithi110.stdout:3026: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:48:04.964 INFO:tasks.rados.rados.0.smithi110.stdout:3024: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:48:04.964 INFO:tasks.rados.rados.0.smithi110.stdout:3024: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:48:04.964 INFO:tasks.rados.rados.0.smithi110.stdout:3024: finishing rollback tid 2 to smithi11075562-30 2024-09-14T03:48:04.964 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1839 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3022: finishing write tid 2 to smithi11075562-1 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3026: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3022: finishing write tid 3 to smithi11075562-1 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3026: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1832 (ObjNum 894 snap 291 seq_num 894) dirty exists 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3022: finishing write tid 4 to smithi11075562-1 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2001 (ObjNum 931 snap 302 seq_num 931) dirty exists 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3022: left oid 1 (ObjNum 931 snap 302 seq_num 931) 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3023: finishing copy_from racing read to smithi11075562-48 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3023: finishing copy_from to smithi11075562-48 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1885 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3027: finishing write tid 1 to smithi11075562-6 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3022: done (6 left) 2024-09-14T03:48:04.965 INFO:tasks.rados.rados.0.smithi110.stdout:3023: done (5 left) 2024-09-14T03:48:04.966 INFO:tasks.rados.rados.0.smithi110.stdout:3024: done (4 left) 2024-09-14T03:48:04.966 INFO:tasks.rados.rados.0.smithi110.stdout:3025: done (3 left) 2024-09-14T03:48:04.966 INFO:tasks.rados.rados.0.smithi110.stdout:3026: done (2 left) 2024-09-14T03:48:04.966 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 302 2024-09-14T03:48:04.966 INFO:tasks.rados.rados.0.smithi110.stdout:3029: seq_num 933 ranges {9150464=434176,9584640=548864,10133504=253952} 2024-09-14T03:48:05.014 INFO:tasks.rados.rados.0.smithi110.stdout:3029: writing smithi11075562-42 from 9150464 to 9584640 tid 1 2024-09-14T03:48:05.017 INFO:tasks.rados.rados.0.smithi110.stdout:3029: writing smithi11075562-42 from 9584640 to 10133504 tid 2 2024-09-14T03:48:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:3029: writing smithi11075562-42 from 10133504 to 10387456 tid 3 2024-09-14T03:48:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 302 2024-09-14T03:48:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:3027: finishing write tid 2 to smithi11075562-6 2024-09-14T03:48:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:3028: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:48:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:3028: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:48:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:3028: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:48:05.019 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1836 (ObjNum 855 snap 278 seq_num 855) dirty exists 2024-09-14T03:48:05.020 INFO:tasks.rados.rados.0.smithi110.stdout:3027: finishing write tid 3 to smithi11075562-6 2024-09-14T03:48:05.020 INFO:tasks.rados.rados.0.smithi110.stdout:3027: finishing write tid 4 to smithi11075562-6 2024-09-14T03:48:05.020 INFO:tasks.rados.rados.0.smithi110.stdout:3030: seq_num 934 ranges {671744=450560,1122304=434176,1556480=368640} 2024-09-14T03:48:05.025 INFO:tasks.rados.rados.0.smithi110.stdout:3030: writing smithi11075562-12 from 671744 to 1122304 tid 1 2024-09-14T03:48:05.027 INFO:tasks.rados.rados.0.smithi110.stdout:3030: writing smithi11075562-12 from 1122304 to 1556480 tid 2 2024-09-14T03:48:05.029 INFO:tasks.rados.rados.0.smithi110.stdout:3030: writing smithi11075562-12 from 1556480 to 1925120 tid 3 2024-09-14T03:48:05.029 INFO:tasks.rados.rados.0.smithi110.stdout:3028: done (3 left) 2024-09-14T03:48:05.030 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 2 current snap is 302 2024-09-14T03:48:05.030 INFO:tasks.rados.rados.0.smithi110.stdout:3031: seq_num 935 ranges {294912=466944,761856=720896,1482752=81920} 2024-09-14T03:48:05.033 INFO:tasks.rados.rados.0.smithi110.stdout:3031: writing smithi11075562-2 from 294912 to 761856 tid 1 2024-09-14T03:48:05.038 INFO:tasks.rados.rados.0.smithi110.stdout:3031: writing smithi11075562-2 from 761856 to 1482752 tid 2 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3031: writing smithi11075562-2 from 1482752 to 1564672 tid 3 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3032: read oid 41 snap -1 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3032: expect (ObjNum 855 snap 278 seq_num 855) 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3033: read oid 36 snap -1 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3033: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3034: copy_from oid 8 from oid 22 current snap is 302 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3027: finishing write tid 5 to smithi11075562-6 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 1571 (ObjNum 932 snap 302 seq_num 932) dirty exists 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3027: left oid 6 (ObjNum 932 snap 302 seq_num 932) 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3029: finishing write tid 1 to smithi11075562-42 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3029: finishing write tid 2 to smithi11075562-42 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3029: finishing write tid 3 to smithi11075562-42 2024-09-14T03:48:05.040 INFO:tasks.rados.rados.0.smithi110.stdout:3029: finishing write tid 4 to smithi11075562-42 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3029: finishing write tid 5 to smithi11075562-42 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2005 (ObjNum 933 snap 302 seq_num 933) dirty exists 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3029: left oid 42 (ObjNum 933 snap 302 seq_num 933) 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3030: finishing write tid 1 to smithi11075562-12 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3030: finishing write tid 2 to smithi11075562-12 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3030: finishing write tid 3 to smithi11075562-12 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3027: done (6 left) 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3029: done (5 left) 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3035: read oid 6 snap -1 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3035: expect (ObjNum 932 snap 302 seq_num 932) 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3036: copy_from oid 23 from oid 48 current snap is 302 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3030: finishing write tid 4 to smithi11075562-12 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:3030: finishing write tid 5 to smithi11075562-12 2024-09-14T03:48:05.041 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2108 (ObjNum 934 snap 302 seq_num 934) dirty exists 2024-09-14T03:48:05.042 INFO:tasks.rados.rados.0.smithi110.stdout:3030: left oid 12 (ObjNum 934 snap 302 seq_num 934) 2024-09-14T03:48:05.042 INFO:tasks.rados.rados.0.smithi110.stdout:3031: finishing write tid 1 to smithi11075562-2 2024-09-14T03:48:05.042 INFO:tasks.rados.rados.0.smithi110.stdout:3030: done (6 left) 2024-09-14T03:48:05.042 INFO:tasks.rados.rados.0.smithi110.stdout:3037: copy_from oid 21 from oid 17 current snap is 302 2024-09-14T03:48:05.042 INFO:tasks.rados.rados.0.smithi110.stdout:3038: snap_remove snap 300 2024-09-14T03:48:05.790 INFO:tasks.rados.rados.0.smithi110.stdout:3038: done (7 left) 2024-09-14T03:48:05.790 INFO:tasks.rados.rados.0.smithi110.stdout:3039: read oid 31 snap -1 2024-09-14T03:48:05.790 INFO:tasks.rados.rados.0.smithi110.stdout:3039: expect (ObjNum 926 snap 301 seq_num 926) 2024-09-14T03:48:05.790 INFO:tasks.rados.rados.0.smithi110.stdout:3040: read oid 42 snap 283 2024-09-14T03:48:05.790 INFO:tasks.rados.rados.0.smithi110.stdout:3040: expect (ObjNum 879 snap 283 seq_num 879) 2024-09-14T03:48:05.790 INFO:tasks.rados.rados.0.smithi110.stdout:3041: setattr oid 9 current snap is 302 2024-09-14T03:48:05.791 INFO:tasks.rados.rados.0.smithi110.stdout:3033: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:3034: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:3031: finishing write tid 2 to smithi11075562-2 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:3031: finishing write tid 3 to smithi11075562-2 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:3031: finishing write tid 4 to smithi11075562-2 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:3031: finishing write tid 5 to smithi11075562-2 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 1674 (ObjNum 935 snap 302 seq_num 935) dirty exists 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:3031: left oid 2 (ObjNum 935 snap 302 seq_num 935) 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:3034: finishing copy_from to smithi11075562-8 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1833 (ObjNum 279 snap 21934 seq_num 304) dirty exists 2024-09-14T03:48:05.800 INFO:tasks.rados.rados.0.smithi110.stdout:3032: expect (ObjNum 855 snap 278 seq_num 855) 2024-09-14T03:48:05.868 INFO:tasks.rados.rados.0.smithi110.stdout:3035: expect (ObjNum 932 snap 302 seq_num 932) 2024-09-14T03:48:06.139 INFO:tasks.rados.rados.0.smithi110.stdout:3037: finishing copy_from racing read to smithi11075562-21 2024-09-14T03:48:06.139 INFO:tasks.rados.rados.0.smithi110.stdout:3037: finishing copy_from to smithi11075562-21 2024-09-14T03:48:06.139 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 1885 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:48:06.139 INFO:tasks.rados.rados.0.smithi110.stdout:3036: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:48:06.139 INFO:tasks.rados.rados.0.smithi110.stdout:3036: finishing copy_from to smithi11075562-23 2024-09-14T03:48:06.139 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1885 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:48:06.139 INFO:tasks.rados.rados.0.smithi110.stdout:3039: expect (ObjNum 926 snap 301 seq_num 926) 2024-09-14T03:48:06.167 INFO:tasks.rados.rados.0.smithi110.stdout:3040: expect (ObjNum 879 snap 283 seq_num 879) 2024-09-14T03:48:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:05 smithi110 ceph-mon[88262]: pgmap v651: 121 pgs: 121 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-14T03:48:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:05 smithi110 ceph-mon[88262]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T03:48:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:05 smithi110 ceph-mon[86423]: pgmap v651: 121 pgs: 121 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-14T03:48:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:05 smithi110 ceph-mon[86423]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T03:48:06.197 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:48:05 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:48:05.800+0000 7fba38156640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 17b after trimming was started 2024-09-14T03:48:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:05 smithi195 ceph-mon[82388]: pgmap v651: 121 pgs: 121 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-14T03:48:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:05 smithi195 ceph-mon[82388]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T03:48:07.072 INFO:tasks.rados.rados.0.smithi110.stdout:3031: done (9 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3032: done (8 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3033: done (7 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3034: done (6 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3035: done (5 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3036: done (4 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3037: done (3 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3039: done (2 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3040: done (1 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3042: rollback oid 31 current snap is 302 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 283 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3043: rmattr oid 21 current snap is 302 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3043: done (2 left) 2024-09-14T03:48:07.073 INFO:tasks.rados.rados.0.smithi110.stdout:3044: copy_from oid 10 from oid 35 current snap is 302 2024-09-14T03:48:07.074 INFO:tasks.rados.rados.0.smithi110.stdout:3045: snap_create 2024-09-14T03:48:07.074 INFO:tasks.rados.rados.0.smithi110.stdout:3044: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:48:07.075 INFO:tasks.rados.rados.0.smithi110.stdout:3042: finishing rollback tid 0 to smithi11075562-31 2024-09-14T03:48:07.075 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 1991 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:48:07.075 INFO:tasks.rados.rados.0.smithi110.stdout:3042: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:48:07.075 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1843 (ObjNum 33 snap 0 seq_num 65163392) dirty dne 2024-09-14T03:48:07.076 INFO:tasks.rados.rados.0.smithi110.stdout:3044: finishing copy_from to smithi11075562-10 2024-09-14T03:48:07.076 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1838 (ObjNum 2673 snap 0 seq_num 2917114464) dirty exists 2024-09-14T03:48:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:06 smithi110 ceph-mon[88262]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T03:48:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:06 smithi110 ceph-mon[86423]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T03:48:07.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:06 smithi195 ceph-mon[82388]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T03:48:07.800 INFO:tasks.rados.rados.0.smithi110.stdout:3041: done (3 left) 2024-09-14T03:48:07.800 INFO:tasks.rados.rados.0.smithi110.stdout:3042: done (2 left) 2024-09-14T03:48:07.800 INFO:tasks.rados.rados.0.smithi110.stdout:3044: done (1 left) 2024-09-14T03:48:07.800 INFO:tasks.rados.rados.0.smithi110.stdout:3045: done (0 left) 2024-09-14T03:48:07.800 INFO:tasks.rados.rados.0.smithi110.stdout:3046: read oid 2 snap 292 2024-09-14T03:48:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:3046: expect (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:48:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:3047: copy_from oid 8 from oid 20 current snap is 303 2024-09-14T03:48:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 303 2024-09-14T03:48:07.801 INFO:tasks.rados.rados.0.smithi110.stdout:3048: seq_num 936 ranges {3350528=483328,3833856=655360,4489216=147456} 2024-09-14T03:48:07.821 INFO:tasks.rados.rados.0.smithi110.stdout:3048: writing smithi11075562-5 from 3350528 to 3833856 tid 1 2024-09-14T03:48:07.825 INFO:tasks.rados.rados.0.smithi110.stdout:3048: writing smithi11075562-5 from 3833856 to 4489216 tid 2 2024-09-14T03:48:07.826 INFO:tasks.rados.rados.0.smithi110.stdout:3048: writing smithi11075562-5 from 4489216 to 4636672 tid 3 2024-09-14T03:48:07.826 INFO:tasks.rados.rados.0.smithi110.stdout:3049: read oid 7 snap -1 2024-09-14T03:48:07.826 INFO:tasks.rados.rados.0.smithi110.stdout:3049: expect (ObjNum 816 snap 270 seq_num 816) 2024-09-14T03:48:07.826 INFO:tasks.rados.rados.0.smithi110.stdout:3050: snap_create 2024-09-14T03:48:07.827 INFO:tasks.rados.rados.0.smithi110.stdout:3046: expect (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:48:07.834 INFO:tasks.rados.rados.0.smithi110.stdout:3047: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:48:07.834 INFO:tasks.rados.rados.0.smithi110.stdout:3047: finishing copy_from to smithi11075562-8 2024-09-14T03:48:07.834 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 1888 (ObjNum 925 snap 301 seq_num 925) dirty exists 2024-09-14T03:48:07.834 INFO:tasks.rados.rados.0.smithi110.stdout:3049: expect (ObjNum 816 snap 270 seq_num 816) 2024-09-14T03:48:07.895 INFO:tasks.rados.rados.0.smithi110.stdout:3048: finishing write tid 1 to smithi11075562-5 2024-09-14T03:48:07.895 INFO:tasks.rados.rados.0.smithi110.stdout:3048: finishing write tid 2 to smithi11075562-5 2024-09-14T03:48:07.895 INFO:tasks.rados.rados.0.smithi110.stdout:3048: finishing write tid 3 to smithi11075562-5 2024-09-14T03:48:07.895 INFO:tasks.rados.rados.0.smithi110.stdout:3048: finishing write tid 4 to smithi11075562-5 2024-09-14T03:48:07.896 INFO:tasks.rados.rados.0.smithi110.stdout:3048: finishing write tid 5 to smithi11075562-5 2024-09-14T03:48:07.896 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2115 (ObjNum 936 snap 303 seq_num 936) dirty exists 2024-09-14T03:48:07.896 INFO:tasks.rados.rados.0.smithi110.stdout:3048: left oid 5 (ObjNum 936 snap 303 seq_num 936) 2024-09-14T03:48:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:07 smithi110 ceph-mon[88262]: pgmap v654: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 262 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 6.0 MiB/s wr, 15 op/s 2024-09-14T03:48:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:07 smithi110 ceph-mon[86423]: pgmap v654: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 262 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 6.0 MiB/s wr, 15 op/s 2024-09-14T03:48:08.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:07 smithi195 ceph-mon[82388]: pgmap v654: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 262 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 6.0 MiB/s wr, 15 op/s 2024-09-14T03:48:08.786 INFO:tasks.rados.rados.0.smithi110.stdout:3046: done (4 left) 2024-09-14T03:48:08.786 INFO:tasks.rados.rados.0.smithi110.stdout:3047: done (3 left) 2024-09-14T03:48:08.786 INFO:tasks.rados.rados.0.smithi110.stdout:3048: done (2 left) 2024-09-14T03:48:08.786 INFO:tasks.rados.rados.0.smithi110.stdout:3049: done (1 left) 2024-09-14T03:48:08.787 INFO:tasks.rados.rados.0.smithi110.stdout:3050: done (0 left) 2024-09-14T03:48:08.787 INFO:tasks.rados.rados.0.smithi110.stdout:3051: delete oid 40 current snap is 304 2024-09-14T03:48:08.790 INFO:tasks.rados.rados.0.smithi110.stdout:3051: done (0 left) 2024-09-14T03:48:08.790 INFO:tasks.rados.rados.0.smithi110.stdout:3052: snap_remove snap 302 2024-09-14T03:48:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:08 smithi110 ceph-mon[88262]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T03:48:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:08 smithi110 ceph-mon[88262]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T03:48:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:08 smithi110 ceph-mon[86423]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T03:48:09.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:08 smithi110 ceph-mon[86423]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T03:48:09.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:08 smithi195 ceph-mon[82388]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T03:48:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:08 smithi195 ceph-mon[82388]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T03:48:09.785 INFO:tasks.rados.rados.0.smithi110.stdout:3052: done (0 left) 2024-09-14T03:48:09.785 INFO:tasks.rados.rados.0.smithi110.stdout:3053: rollback oid 38 current snap is 304 2024-09-14T03:48:09.785 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 303 2024-09-14T03:48:09.785 INFO:tasks.rados.rados.0.smithi110.stdout:3054: setattr oid 47 current snap is 304 2024-09-14T03:48:09.788 INFO:tasks.rados.rados.0.smithi110.stdout:3055: copy_from oid 15 from oid 33 current snap is 304 2024-09-14T03:48:09.788 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 304 2024-09-14T03:48:09.788 INFO:tasks.rados.rados.0.smithi110.stdout:3056: seq_num 938 ranges {0=516096,516096=417792,933888=385024} 2024-09-14T03:48:09.791 INFO:tasks.rados.rados.0.smithi110.stdout:3056: writing smithi11075562-35 from 0 to 516096 tid 1 2024-09-14T03:48:09.794 INFO:tasks.rados.rados.0.smithi110.stdout:3056: writing smithi11075562-35 from 516096 to 933888 tid 2 2024-09-14T03:48:09.797 INFO:tasks.rados.rados.0.smithi110.stdout:3056: writing smithi11075562-35 from 933888 to 1318912 tid 3 2024-09-14T03:48:09.797 INFO:tasks.rados.rados.0.smithi110.stdout:3057: copy_from oid 4 from oid 5 current snap is 304 2024-09-14T03:48:09.797 INFO:tasks.rados.rados.0.smithi110.stdout:3053: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:48:09.797 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1612 (ObjNum 49 snap 0 seq_num 43186256) dirty dne 2024-09-14T03:48:09.797 INFO:tasks.rados.rados.0.smithi110.stdout:3053: done (4 left) 2024-09-14T03:48:09.797 INFO:tasks.rados.rados.0.smithi110.stdout:3058: delete oid 48 current snap is 304 2024-09-14T03:48:09.798 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 2117 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:48:09.801 INFO:tasks.rados.rados.0.smithi110.stdout:3054: done (4 left) 2024-09-14T03:48:09.802 INFO:tasks.rados.rados.0.smithi110.stdout:3058: done (3 left) 2024-09-14T03:48:09.802 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 304 2024-09-14T03:48:09.802 INFO:tasks.rados.rados.0.smithi110.stdout:3059: seq_num 940 ranges {7544832=548864,8093696=589824,8683520=212992} 2024-09-14T03:48:09.846 INFO:tasks.rados.rados.0.smithi110.stdout:3059: writing smithi11075562-44 from 7544832 to 8093696 tid 1 2024-09-14T03:48:09.849 INFO:tasks.rados.rados.0.smithi110.stdout:3059: writing smithi11075562-44 from 8093696 to 8683520 tid 2 2024-09-14T03:48:09.851 INFO:tasks.rados.rados.0.smithi110.stdout:3059: writing smithi11075562-44 from 8683520 to 8896512 tid 3 2024-09-14T03:48:09.851 INFO:tasks.rados.rados.0.smithi110.stdout:3060: read oid 11 snap -1 2024-09-14T03:48:09.851 INFO:tasks.rados.rados.0.smithi110.stdout:3060: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:09.851 INFO:tasks.rados.rados.0.smithi110.stdout:3061: delete oid 31 current snap is 304 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3055: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3056: finishing write tid 1 to smithi11075562-35 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3055: finishing copy_from to smithi11075562-15 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1990 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3056: finishing write tid 2 to smithi11075562-35 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3056: finishing write tid 3 to smithi11075562-35 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3056: finishing write tid 4 to smithi11075562-35 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3056: finishing write tid 5 to smithi11075562-35 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 2009 (ObjNum 938 snap 304 seq_num 938) dirty exists 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3056: left oid 35 (ObjNum 938 snap 304 seq_num 938) 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3057: finishing copy_from racing read to smithi11075562-4 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3057: finishing copy_from to smithi11075562-4 2024-09-14T03:48:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2115 (ObjNum 936 snap 303 seq_num 936) dirty exists 2024-09-14T03:48:09.853 INFO:tasks.rados.rados.0.smithi110.stdout:3059: finishing write tid 1 to smithi11075562-44 2024-09-14T03:48:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:3055: done (5 left) 2024-09-14T03:48:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:3056: done (4 left) 2024-09-14T03:48:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:3057: done (3 left) 2024-09-14T03:48:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:3061: done (2 left) 2024-09-14T03:48:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:3062: read oid 36 snap -1 2024-09-14T03:48:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:3062: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:3063: snap_create 2024-09-14T03:48:09.854 INFO:tasks.rados.rados.0.smithi110.stdout:3059: finishing write tid 2 to smithi11075562-44 2024-09-14T03:48:09.855 INFO:tasks.rados.rados.0.smithi110.stdout:3059: finishing write tid 3 to smithi11075562-44 2024-09-14T03:48:09.855 INFO:tasks.rados.rados.0.smithi110.stdout:3059: finishing write tid 4 to smithi11075562-44 2024-09-14T03:48:09.855 INFO:tasks.rados.rados.0.smithi110.stdout:3062: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3059: finishing write tid 5 to smithi11075562-44 2024-09-14T03:48:09.862 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 1813 (ObjNum 940 snap 304 seq_num 940) dirty exists 2024-09-14T03:48:09.862 INFO:tasks.rados.rados.0.smithi110.stdout:3059: left oid 44 (ObjNum 940 snap 304 seq_num 940) 2024-09-14T03:48:09.862 INFO:tasks.rados.rados.0.smithi110.stdout:3060: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:10.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:09 smithi110 ceph-mon[86423]: pgmap v656: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 274 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.4 MiB/s wr, 35 op/s 2024-09-14T03:48:10.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:09 smithi110 ceph-mon[86423]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T03:48:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:09 smithi110 ceph-mon[88262]: pgmap v656: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 274 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.4 MiB/s wr, 35 op/s 2024-09-14T03:48:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:09 smithi110 ceph-mon[88262]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T03:48:10.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:09 smithi195 ceph-mon[82388]: pgmap v656: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 274 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.4 MiB/s wr, 35 op/s 2024-09-14T03:48:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:09 smithi195 ceph-mon[82388]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T03:48:10.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:48:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:48:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3059: done (3 left) 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3060: done (2 left) 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3062: done (1 left) 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3063: done (0 left) 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3064: rollback oid 49 current snap is 305 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 301 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3065: read oid 47 snap -1 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3065: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3066: read oid 40 snap -1 2024-09-14T03:48:10.814 INFO:tasks.rados.rados.0.smithi110.stdout:3066: expect deleted 2024-09-14T03:48:10.815 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 305 2024-09-14T03:48:10.815 INFO:tasks.rados.rados.0.smithi110.stdout:3067: seq_num 942 ranges {0=581632,581632=761856,1343488=40960} 2024-09-14T03:48:10.817 INFO:tasks.rados.rados.0.smithi110.stdout:3067: writing smithi11075562-28 from 0 to 581632 tid 1 2024-09-14T03:48:10.822 INFO:tasks.rados.rados.0.smithi110.stdout:3067: writing smithi11075562-28 from 581632 to 1343488 tid 2 2024-09-14T03:48:10.822 INFO:tasks.rados.rados.0.smithi110.stdout:3067: writing smithi11075562-28 from 1343488 to 1384448 tid 3 2024-09-14T03:48:10.822 INFO:tasks.rados.rados.0.smithi110.stdout:3068: read oid 12 snap -1 2024-09-14T03:48:10.822 INFO:tasks.rados.rados.0.smithi110.stdout:3068: expect (ObjNum 934 snap 302 seq_num 934) 2024-09-14T03:48:10.822 INFO:tasks.rados.rados.0.smithi110.stdout:3069: snap_remove snap 283 2024-09-14T03:48:11.818 INFO:tasks.rados.rados.0.smithi110.stdout:3069: done (5 left) 2024-09-14T03:48:11.818 INFO:tasks.rados.rados.0.smithi110.stdout:3070: delete oid 41 current snap is 305 2024-09-14T03:48:11.818 INFO:tasks.rados.rados.0.smithi110.stdout:3065: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:11.826 INFO:tasks.rados.rados.0.smithi110.stdout:3067: finishing write tid 1 to smithi11075562-28 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3064: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3064: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3064: finishing rollback tid 2 to smithi11075562-49 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2012 (ObjNum 907 snap 297 seq_num 907) dirty exists 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3067: finishing write tid 2 to smithi11075562-28 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3067: finishing write tid 3 to smithi11075562-28 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3067: finishing write tid 4 to smithi11075562-28 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3067: finishing write tid 5 to smithi11075562-28 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 1678 (ObjNum 942 snap 305 seq_num 942) dirty exists 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3067: left oid 28 (ObjNum 942 snap 305 seq_num 942) 2024-09-14T03:48:11.827 INFO:tasks.rados.rados.0.smithi110.stdout:3068: expect (ObjNum 934 snap 302 seq_num 934) 2024-09-14T03:48:11.894 INFO:tasks.rados.rados.0.smithi110.stdout:3064: done (5 left) 2024-09-14T03:48:11.895 INFO:tasks.rados.rados.0.smithi110.stdout:3065: done (4 left) 2024-09-14T03:48:11.895 INFO:tasks.rados.rados.0.smithi110.stdout:3066: done (3 left) 2024-09-14T03:48:11.895 INFO:tasks.rados.rados.0.smithi110.stdout:3067: done (2 left) 2024-09-14T03:48:11.895 INFO:tasks.rados.rados.0.smithi110.stdout:3068: done (1 left) 2024-09-14T03:48:11.895 INFO:tasks.rados.rados.0.smithi110.stdout:3070: done (0 left) 2024-09-14T03:48:11.895 INFO:tasks.rados.rados.0.smithi110.stdout:3071: snap_remove snap 304 2024-09-14T03:48:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:11 smithi110 ceph-mon[88262]: pgmap v659: 121 pgs: 121 active+clean; 298 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 8.4 MiB/s wr, 34 op/s 2024-09-14T03:48:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:11 smithi110 ceph-mon[88262]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T03:48:11.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:48:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:48:11] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-14T03:48:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:11 smithi110 ceph-mon[86423]: pgmap v659: 121 pgs: 121 active+clean; 298 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 8.4 MiB/s wr, 34 op/s 2024-09-14T03:48:11.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:11 smithi110 ceph-mon[86423]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T03:48:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:11 smithi195 ceph-mon[82388]: pgmap v659: 121 pgs: 121 active+clean; 298 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 8.4 MiB/s wr, 34 op/s 2024-09-14T03:48:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:11 smithi195 ceph-mon[82388]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T03:48:12.823 INFO:tasks.rados.rados.0.smithi110.stdout:3071: done (0 left) 2024-09-14T03:48:12.824 INFO:tasks.rados.rados.0.smithi110.stdout:3072: copy_from oid 33 from oid 15 current snap is 305 2024-09-14T03:48:12.824 INFO:tasks.rados.rados.0.smithi110.stdout:3073: copy_from oid 16 from oid 12 current snap is 305 2024-09-14T03:48:12.824 INFO:tasks.rados.rados.0.smithi110.stdout:3074: rmattr oid 8 current snap is 305 2024-09-14T03:48:12.824 INFO:tasks.rados.rados.0.smithi110.stdout:3074: done (2 left) 2024-09-14T03:48:12.824 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 305 2024-09-14T03:48:12.824 INFO:tasks.rados.rados.0.smithi110.stdout:3075: seq_num 944 ranges {4636672=614400,5251072=573440,5824512=229376} 2024-09-14T03:48:12.852 INFO:tasks.rados.rados.0.smithi110.stdout:3075: writing smithi11075562-4 from 4636672 to 5251072 tid 1 2024-09-14T03:48:12.856 INFO:tasks.rados.rados.0.smithi110.stdout:3075: writing smithi11075562-4 from 5251072 to 5824512 tid 2 2024-09-14T03:48:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:3075: writing smithi11075562-4 from 5824512 to 6053888 tid 3 2024-09-14T03:48:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 305 2024-09-14T03:48:12.857 INFO:tasks.rados.rados.0.smithi110.stdout:3073: finishing copy_from racing read to smithi11075562-16 2024-09-14T03:48:12.858 INFO:tasks.rados.rados.0.smithi110.stdout:3076: seq_num 945 ranges {0=630784,630784=458752,1089536=344064} 2024-09-14T03:48:12.861 INFO:tasks.rados.rados.0.smithi110.stdout:3076: writing smithi11075562-36 from 0 to 630784 tid 1 2024-09-14T03:48:12.863 INFO:tasks.rados.rados.0.smithi110.stdout:3076: writing smithi11075562-36 from 630784 to 1089536 tid 2 2024-09-14T03:48:12.866 INFO:tasks.rados.rados.0.smithi110.stdout:3076: writing smithi11075562-36 from 1089536 to 1433600 tid 3 2024-09-14T03:48:12.866 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 305 2024-09-14T03:48:12.866 INFO:tasks.rados.rados.0.smithi110.stdout:3073: finishing copy_from to smithi11075562-16 2024-09-14T03:48:12.866 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2108 (ObjNum 934 snap 302 seq_num 934) dirty exists 2024-09-14T03:48:12.866 INFO:tasks.rados.rados.0.smithi110.stdout:3072: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:48:12.866 INFO:tasks.rados.rados.0.smithi110.stdout:3072: finishing copy_from to smithi11075562-33 2024-09-14T03:48:12.866 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1990 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:48:12.866 INFO:tasks.rados.rados.0.smithi110.stdout:3075: finishing write tid 1 to smithi11075562-4 2024-09-14T03:48:12.867 INFO:tasks.rados.rados.0.smithi110.stdout:3077: seq_num 946 ranges {5283840=647168,5931008=745472,6676480=57344} 2024-09-14T03:48:12.897 INFO:tasks.rados.rados.0.smithi110.stdout:3077: writing smithi11075562-6 from 5283840 to 5931008 tid 1 2024-09-14T03:48:12.901 INFO:tasks.rados.rados.0.smithi110.stdout:3077: writing smithi11075562-6 from 5931008 to 6676480 tid 2 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:3077: writing smithi11075562-6 from 6676480 to 6733824 tid 3 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:3072: done (4 left) 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:3073: done (3 left) 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:3078: snap_create 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:3075: finishing write tid 2 to smithi11075562-4 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:3075: finishing write tid 3 to smithi11075562-4 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:3075: finishing write tid 4 to smithi11075562-4 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:3075: finishing write tid 5 to smithi11075562-4 2024-09-14T03:48:12.902 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2119 (ObjNum 944 snap 305 seq_num 944) dirty exists 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:3075: left oid 4 (ObjNum 944 snap 305 seq_num 944) 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:3076: finishing write tid 1 to smithi11075562-36 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:3076: finishing write tid 2 to smithi11075562-36 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:3076: finishing write tid 3 to smithi11075562-36 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:3076: finishing write tid 4 to smithi11075562-36 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:3076: finishing write tid 5 to smithi11075562-36 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2134 (ObjNum 945 snap 305 seq_num 945) dirty exists 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:3076: left oid 36 (ObjNum 945 snap 305 seq_num 945) 2024-09-14T03:48:12.903 INFO:tasks.rados.rados.0.smithi110.stdout:3077: finishing write tid 1 to smithi11075562-6 2024-09-14T03:48:12.906 INFO:tasks.rados.rados.0.smithi110.stdout:3077: finishing write tid 2 to smithi11075562-6 2024-09-14T03:48:12.906 INFO:tasks.rados.rados.0.smithi110.stdout:3077: finishing write tid 3 to smithi11075562-6 2024-09-14T03:48:12.906 INFO:tasks.rados.rados.0.smithi110.stdout:3077: finishing write tid 4 to smithi11075562-6 2024-09-14T03:48:12.907 INFO:tasks.rados.rados.0.smithi110.stdout:3077: finishing write tid 5 to smithi11075562-6 2024-09-14T03:48:12.907 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2112 (ObjNum 946 snap 305 seq_num 946) dirty exists 2024-09-14T03:48:12.907 INFO:tasks.rados.rados.0.smithi110.stdout:3077: left oid 6 (ObjNum 946 snap 305 seq_num 946) 2024-09-14T03:48:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:12 smithi195 ceph-mon[82388]: osdmap e688: 8 total, 8 up, 8 in 2024-09-14T03:48:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:12 smithi195 ceph-mon[82388]: pgmap v662: 121 pgs: 121 active+clean; 302 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.6 MiB/s wr, 22 op/s 2024-09-14T03:48:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:12 smithi110 ceph-mon[86423]: osdmap e688: 8 total, 8 up, 8 in 2024-09-14T03:48:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:12 smithi110 ceph-mon[86423]: pgmap v662: 121 pgs: 121 active+clean; 302 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.6 MiB/s wr, 22 op/s 2024-09-14T03:48:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:12 smithi110 ceph-mon[88262]: osdmap e688: 8 total, 8 up, 8 in 2024-09-14T03:48:13.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:12 smithi110 ceph-mon[88262]: pgmap v662: 121 pgs: 121 active+clean; 302 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.6 MiB/s wr, 22 op/s 2024-09-14T03:48:13.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:13.783 INFO:tasks.rados.rados.0.smithi110.stdout:3075: done (3 left) 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3076: done (2 left) 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3077: done (1 left) 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3078: done (0 left) 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3079: rmattr oid 44 current snap is 306 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3080: rmattr oid 36 current snap is 306 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3081: read oid 23 snap -1 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3081: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3082: rmattr oid 5 current snap is 306 2024-09-14T03:48:13.784 INFO:tasks.rados.rados.0.smithi110.stdout:3083: snap_create 2024-09-14T03:48:13.796 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2136 (ObjNum 945 snap 305 seq_num 945) dirty exists 2024-09-14T03:48:13.797 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2138 (ObjNum 936 snap 303 seq_num 936) dirty exists 2024-09-14T03:48:13.799 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 1814 (ObjNum 940 snap 304 seq_num 940) dirty exists 2024-09-14T03:48:13.802 INFO:tasks.rados.rados.0.smithi110.stdout:3081: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:13 smithi110 ceph-mon[86423]: osdmap e689: 8 total, 8 up, 8 in 2024-09-14T03:48:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:13 smithi110 ceph-mon[86423]: osdmap e690: 8 total, 8 up, 8 in 2024-09-14T03:48:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:13 smithi110 ceph-mon[88262]: osdmap e689: 8 total, 8 up, 8 in 2024-09-14T03:48:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:13 smithi110 ceph-mon[88262]: osdmap e690: 8 total, 8 up, 8 in 2024-09-14T03:48:14.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:13 smithi195 ceph-mon[82388]: osdmap e689: 8 total, 8 up, 8 in 2024-09-14T03:48:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:13 smithi195 ceph-mon[82388]: osdmap e690: 8 total, 8 up, 8 in 2024-09-14T03:48:14.785 INFO:tasks.rados.rados.0.smithi110.stdout:3079: done (4 left) 2024-09-14T03:48:14.786 INFO:tasks.rados.rados.0.smithi110.stdout:3080: done (3 left) 2024-09-14T03:48:14.786 INFO:tasks.rados.rados.0.smithi110.stdout:3081: done (2 left) 2024-09-14T03:48:14.786 INFO:tasks.rados.rados.0.smithi110.stdout:3082: done (1 left) 2024-09-14T03:48:14.786 INFO:tasks.rados.rados.0.smithi110.stdout:3083: done (0 left) 2024-09-14T03:48:14.786 INFO:tasks.rados.rados.0.smithi110.stdout:3084: snap_remove snap 301 2024-09-14T03:48:15.794 INFO:tasks.rados.rados.0.smithi110.stdout:3084: done (0 left) 2024-09-14T03:48:15.794 INFO:tasks.rados.rados.0.smithi110.stdout:3085: read oid 1 snap -1 2024-09-14T03:48:15.794 INFO:tasks.rados.rados.0.smithi110.stdout:3085: expect (ObjNum 931 snap 302 seq_num 931) 2024-09-14T03:48:15.795 INFO:tasks.rados.rados.0.smithi110.stdout:3086: read oid 8 snap -1 2024-09-14T03:48:15.795 INFO:tasks.rados.rados.0.smithi110.stdout:3086: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:15.795 INFO:tasks.rados.rados.0.smithi110.stdout:3087: rmattr oid 44 current snap is 307 2024-09-14T03:48:15.795 INFO:tasks.rados.rados.0.smithi110.stdout:3087: done (2 left) 2024-09-14T03:48:15.795 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 307 2024-09-14T03:48:15.795 INFO:tasks.rados.rados.0.smithi110.stdout:3088: seq_num 947 ranges {1925120=663552,2588672=630784,3219456=172032} 2024-09-14T03:48:15.810 INFO:tasks.rados.rados.0.smithi110.stdout:3088: writing smithi11075562-12 from 1925120 to 2588672 tid 1 2024-09-14T03:48:15.815 INFO:tasks.rados.rados.0.smithi110.stdout:3088: writing smithi11075562-12 from 2588672 to 3219456 tid 2 2024-09-14T03:48:15.816 INFO:tasks.rados.rados.0.smithi110.stdout:3088: writing smithi11075562-12 from 3219456 to 3391488 tid 3 2024-09-14T03:48:15.816 INFO:tasks.rados.rados.0.smithi110.stdout:3089: rmattr oid 38 current snap is 307 2024-09-14T03:48:15.816 INFO:tasks.rados.rados.0.smithi110.stdout:3089: done (3 left) 2024-09-14T03:48:15.816 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 307 2024-09-14T03:48:15.817 INFO:tasks.rados.rados.0.smithi110.stdout:3090: seq_num 948 ranges {10190848=679936,10870784=516096,11386880=286720} 2024-09-14T03:48:15.876 INFO:tasks.rados.rados.0.smithi110.stdout:3090: writing smithi11075562-26 from 10190848 to 10870784 tid 1 2024-09-14T03:48:15.878 INFO:tasks.rados.rados.0.smithi110.stdout:3090: writing smithi11075562-26 from 10870784 to 11386880 tid 2 2024-09-14T03:48:15.880 INFO:tasks.rados.rados.0.smithi110.stdout:3090: writing smithi11075562-26 from 11386880 to 11673600 tid 3 2024-09-14T03:48:15.880 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 307 2024-09-14T03:48:15.880 INFO:tasks.rados.rados.0.smithi110.stdout:3086: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:15.948 INFO:tasks.rados.rados.0.smithi110.stdout:3088: finishing write tid 1 to smithi11075562-12 2024-09-14T03:48:15.949 INFO:tasks.rados.rados.0.smithi110.stdout:3088: finishing write tid 2 to smithi11075562-12 2024-09-14T03:48:15.949 INFO:tasks.rados.rados.0.smithi110.stdout:3091: seq_num 949 ranges {0=704512,704512=794624,1499136=8192} 2024-09-14T03:48:15.952 INFO:tasks.rados.rados.0.smithi110.stdout:3091: writing smithi11075562-19 from 0 to 704512 tid 1 2024-09-14T03:48:15.957 INFO:tasks.rados.rados.0.smithi110.stdout:3091: writing smithi11075562-19 from 704512 to 1499136 tid 2 2024-09-14T03:48:15.957 INFO:tasks.rados.rados.0.smithi110.stdout:3091: writing smithi11075562-19 from 1499136 to 1507328 tid 3 2024-09-14T03:48:15.957 INFO:tasks.rados.rados.0.smithi110.stdout:3086: done (4 left) 2024-09-14T03:48:15.958 INFO:tasks.rados.rados.0.smithi110.stdout:3092: delete oid 47 current snap is 307 2024-09-14T03:48:15.958 INFO:tasks.rados.rados.0.smithi110.stdout:3088: finishing write tid 3 to smithi11075562-12 2024-09-14T03:48:15.958 INFO:tasks.rados.rados.0.smithi110.stdout:3088: finishing write tid 4 to smithi11075562-12 2024-09-14T03:48:15.958 INFO:tasks.rados.rados.0.smithi110.stdout:3088: finishing write tid 5 to smithi11075562-12 2024-09-14T03:48:15.958 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2151 (ObjNum 947 snap 307 seq_num 947) dirty exists 2024-09-14T03:48:15.958 INFO:tasks.rados.rados.0.smithi110.stdout:3088: left oid 12 (ObjNum 947 snap 307 seq_num 947) 2024-09-14T03:48:15.958 INFO:tasks.rados.rados.0.smithi110.stdout:3085: expect (ObjNum 931 snap 302 seq_num 931) 2024-09-14T03:48:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:15 smithi110 ceph-mon[88262]: pgmap v665: 121 pgs: 121 active+clean; 305 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-14T03:48:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:15 smithi110 ceph-mon[88262]: osdmap e691: 8 total, 8 up, 8 in 2024-09-14T03:48:16.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:48:15 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:48:15.813+0000 7fba39959640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 17c after trimming was started 2024-09-14T03:48:16.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:15 smithi110 ceph-mon[86423]: pgmap v665: 121 pgs: 121 active+clean; 305 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-14T03:48:16.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:15 smithi110 ceph-mon[86423]: osdmap e691: 8 total, 8 up, 8 in 2024-09-14T03:48:16.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:15 smithi195 ceph-mon[82388]: pgmap v665: 121 pgs: 121 active+clean; 305 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-14T03:48:16.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:15 smithi195 ceph-mon[82388]: osdmap e691: 8 total, 8 up, 8 in 2024-09-14T03:48:16.725 INFO:tasks.rados.rados.0.smithi110.stdout:3090: finishing write tid 1 to smithi11075562-26 2024-09-14T03:48:16.725 INFO:tasks.rados.rados.0.smithi110.stdout:3090: finishing write tid 2 to smithi11075562-26 2024-09-14T03:48:16.725 INFO:tasks.rados.rados.0.smithi110.stdout:3090: finishing write tid 3 to smithi11075562-26 2024-09-14T03:48:16.725 INFO:tasks.rados.rados.0.smithi110.stdout:3090: finishing write tid 4 to smithi11075562-26 2024-09-14T03:48:16.725 INFO:tasks.rados.rados.0.smithi110.stdout:3085: done (4 left) 2024-09-14T03:48:16.726 INFO:tasks.rados.rados.0.smithi110.stdout:3088: done (3 left) 2024-09-14T03:48:16.726 INFO:tasks.rados.rados.0.smithi110.stdout:3092: done (2 left) 2024-09-14T03:48:16.726 INFO:tasks.rados.rados.0.smithi110.stdout:3093: read oid 32 snap -1 2024-09-14T03:48:16.726 INFO:tasks.rados.rados.0.smithi110.stdout:3093: expect (ObjNum 875 snap 281 seq_num 875) 2024-09-14T03:48:16.726 INFO:tasks.rados.rados.0.smithi110.stdout:3094: snap_remove snap 293 2024-09-14T03:48:16.796 INFO:tasks.rados.rados.0.smithi110.stdout:3094: done (3 left) 2024-09-14T03:48:16.796 INFO:tasks.rados.rados.0.smithi110.stdout:3095: read oid 46 snap -1 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3095: expect (ObjNum 907 snap 297 seq_num 907) 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3096: rollback oid 41 current snap is 307 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 303 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3090: finishing write tid 5 to smithi11075562-26 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1836 (ObjNum 948 snap 307 seq_num 948) dirty exists 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3090: left oid 26 (ObjNum 948 snap 307 seq_num 948) 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3091: finishing write tid 1 to smithi11075562-19 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3091: finishing write tid 2 to smithi11075562-19 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3091: finishing write tid 3 to smithi11075562-19 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3091: finishing write tid 4 to smithi11075562-19 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3091: finishing write tid 5 to smithi11075562-19 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2116 (ObjNum 949 snap 307 seq_num 949) dirty exists 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3091: left oid 19 (ObjNum 949 snap 307 seq_num 949) 2024-09-14T03:48:16.797 INFO:tasks.rados.rados.0.smithi110.stdout:3093: expect (ObjNum 875 snap 281 seq_num 875) 2024-09-14T03:48:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:16 smithi110 ceph-mon[88262]: osdmap e692: 8 total, 8 up, 8 in 2024-09-14T03:48:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:16 smithi110 ceph-mon[86423]: osdmap e692: 8 total, 8 up, 8 in 2024-09-14T03:48:17.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:16 smithi195 ceph-mon[82388]: osdmap e692: 8 total, 8 up, 8 in 2024-09-14T03:48:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:17 smithi110 ceph-mon[88262]: pgmap v668: 121 pgs: 121 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:48:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:17 smithi110 ceph-mon[88262]: osdmap e693: 8 total, 8 up, 8 in 2024-09-14T03:48:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:17 smithi110 ceph-mon[86423]: pgmap v668: 121 pgs: 121 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:48:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:17 smithi110 ceph-mon[86423]: osdmap e693: 8 total, 8 up, 8 in 2024-09-14T03:48:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:17 smithi195 ceph-mon[82388]: pgmap v668: 121 pgs: 121 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:48:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:17 smithi195 ceph-mon[82388]: osdmap e693: 8 total, 8 up, 8 in 2024-09-14T03:48:18.734 INFO:tasks.rados.rados.0.smithi110.stdout:3096: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:48:18.734 INFO:tasks.rados.rados.0.smithi110.stdout:3096: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:48:18.734 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1888 (ObjNum 855 snap 278 seq_num 855) dirty exists 2024-09-14T03:48:18.734 INFO:tasks.rados.rados.0.smithi110.stdout:3090: done (4 left) 2024-09-14T03:48:18.734 INFO:tasks.rados.rados.0.smithi110.stdout:3091: done (3 left) 2024-09-14T03:48:18.734 INFO:tasks.rados.rados.0.smithi110.stdout:3093: done (2 left) 2024-09-14T03:48:18.735 INFO:tasks.rados.rados.0.smithi110.stdout:3096: done (1 left) 2024-09-14T03:48:18.735 INFO:tasks.rados.rados.0.smithi110.stdout:3097: snap_remove snap 305 2024-09-14T03:48:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:3097: done (1 left) 2024-09-14T03:48:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:3098: copy_from oid 49 from oid 9 current snap is 307 2024-09-14T03:48:18.785 INFO:tasks.rados.rados.0.smithi110.stdout:3095: expect (ObjNum 907 snap 297 seq_num 907) 2024-09-14T03:48:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:3098: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:48:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:3098: finishing copy_from to smithi11075562-49 2024-09-14T03:48:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1991 (ObjNum 65 snap 0 seq_num 1953787753) dirty exists 2024-09-14T03:48:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:3095: done (1 left) 2024-09-14T03:48:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:3098: done (0 left) 2024-09-14T03:48:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:3099: rollback oid 35 current snap is 307 2024-09-14T03:48:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 303 2024-09-14T03:48:18.841 INFO:tasks.rados.rados.0.smithi110.stdout:3100: snap_remove snap 306 2024-09-14T03:48:19.801 INFO:tasks.rados.rados.0.smithi110.stdout:3099: finishing rollback tid 0 to smithi11075562-35 2024-09-14T03:48:19.801 INFO:tasks.rados.rados.0.smithi110.stdout:3099: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:48:19.801 INFO:tasks.rados.rados.0.smithi110.stdout:3099: finishing rollback tid 2 to smithi11075562-35 2024-09-14T03:48:19.801 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 2015 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:48:19.802 INFO:tasks.rados.rados.0.smithi110.stdout:3099: done (1 left) 2024-09-14T03:48:19.802 INFO:tasks.rados.rados.0.smithi110.stdout:3100: done (0 left) 2024-09-14T03:48:19.802 INFO:tasks.rados.rados.0.smithi110.stdout:3101: delete oid 27 current snap is 307 2024-09-14T03:48:19.806 INFO:tasks.rados.rados.0.smithi110.stdout:3101: done (0 left) 2024-09-14T03:48:19.806 INFO:tasks.rados.rados.0.smithi110.stdout:3102: delete oid 29 current snap is 307 2024-09-14T03:48:19.809 INFO:tasks.rados.rados.0.smithi110.stdout:3102: done (0 left) 2024-09-14T03:48:19.809 INFO:tasks.rados.rados.0.smithi110.stdout:3103: rmattr oid 32 current snap is 307 2024-09-14T03:48:19.809 INFO:tasks.rados.rados.0.smithi110.stdout:3104: snap_remove snap 292 2024-09-14T03:48:20.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:19 smithi110 ceph-mon[88262]: pgmap v670: 121 pgs: 121 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2024-09-14T03:48:20.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:19 smithi110 ceph-mon[88262]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T03:48:20.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:19 smithi110 ceph-mon[86423]: pgmap v670: 121 pgs: 121 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2024-09-14T03:48:20.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:19 smithi110 ceph-mon[86423]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T03:48:20.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:19 smithi195 ceph-mon[82388]: pgmap v670: 121 pgs: 121 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2024-09-14T03:48:20.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:19 smithi195 ceph-mon[82388]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T03:48:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:48:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:48:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:48:20.807 INFO:tasks.rados.rados.0.smithi110.stdout:3104: done (1 left) 2024-09-14T03:48:20.807 INFO:tasks.rados.rados.0.smithi110.stdout:3105: read oid 29 snap -1 2024-09-14T03:48:20.807 INFO:tasks.rados.rados.0.smithi110.stdout:3105: expect deleted 2024-09-14T03:48:20.807 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 307 2024-09-14T03:48:20.808 INFO:tasks.rados.rados.0.smithi110.stdout:3106: seq_num 953 ranges {6733824=770048,7503872=786432,8290304=16384} 2024-09-14T03:48:20.850 INFO:tasks.rados.rados.0.smithi110.stdout:3106: writing smithi11075562-6 from 6733824 to 7503872 tid 1 2024-09-14T03:48:20.855 INFO:tasks.rados.rados.0.smithi110.stdout:3106: writing smithi11075562-6 from 7503872 to 8290304 tid 2 2024-09-14T03:48:20.855 INFO:tasks.rados.rados.0.smithi110.stdout:3106: writing smithi11075562-6 from 8290304 to 8306688 tid 3 2024-09-14T03:48:20.855 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 307 2024-09-14T03:48:20.856 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1991 (ObjNum 875 snap 281 seq_num 875) dirty exists 2024-09-14T03:48:20.856 INFO:tasks.rados.rados.0.smithi110.stdout:3107: seq_num 954 ranges {8896512=786432,9682944=671744,10354688=131072} 2024-09-14T03:48:20.906 INFO:tasks.rados.rados.0.smithi110.stdout:3107: writing smithi11075562-44 from 8896512 to 9682944 tid 1 2024-09-14T03:48:20.909 INFO:tasks.rados.rados.0.smithi110.stdout:3107: writing smithi11075562-44 from 9682944 to 10354688 tid 2 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3107: writing smithi11075562-44 from 10354688 to 10485760 tid 3 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3103: done (3 left) 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3108: read oid 8 snap -1 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3108: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3109: setattr oid 21 current snap is 307 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3106: finishing write tid 1 to smithi11075562-6 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3106: finishing write tid 2 to smithi11075562-6 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3106: finishing write tid 3 to smithi11075562-6 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3106: finishing write tid 4 to smithi11075562-6 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3106: finishing write tid 5 to smithi11075562-6 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2120 (ObjNum 953 snap 307 seq_num 953) dirty exists 2024-09-14T03:48:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3106: left oid 6 (ObjNum 953 snap 307 seq_num 953) 2024-09-14T03:48:20.912 INFO:tasks.rados.rados.0.smithi110.stdout:3107: finishing write tid 1 to smithi11075562-44 2024-09-14T03:48:20.914 INFO:tasks.rados.rados.0.smithi110.stdout:3105: done (4 left) 2024-09-14T03:48:20.914 INFO:tasks.rados.rados.0.smithi110.stdout:3106: done (3 left) 2024-09-14T03:48:20.914 INFO:tasks.rados.rados.0.smithi110.stdout:3110: snap_create 2024-09-14T03:48:20.914 INFO:tasks.rados.rados.0.smithi110.stdout:3107: finishing write tid 2 to smithi11075562-44 2024-09-14T03:48:20.914 INFO:tasks.rados.rados.0.smithi110.stdout:3107: finishing write tid 3 to smithi11075562-44 2024-09-14T03:48:20.914 INFO:tasks.rados.rados.0.smithi110.stdout:3107: finishing write tid 4 to smithi11075562-44 2024-09-14T03:48:20.915 INFO:tasks.rados.rados.0.smithi110.stdout:3107: finishing write tid 5 to smithi11075562-44 2024-09-14T03:48:20.915 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 1818 (ObjNum 954 snap 307 seq_num 954) dirty exists 2024-09-14T03:48:20.915 INFO:tasks.rados.rados.0.smithi110.stdout:3107: left oid 44 (ObjNum 954 snap 307 seq_num 954) 2024-09-14T03:48:20.917 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 1889 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:48:20.918 INFO:tasks.rados.rados.0.smithi110.stdout:3108: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:20 smithi110 ceph-mon[88262]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T03:48:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:20 smithi110 ceph-mon[86423]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T03:48:21.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:20 smithi195 ceph-mon[82388]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T03:48:21.811 INFO:tasks.rados.rados.0.smithi110.stdout:3107: done (3 left) 2024-09-14T03:48:21.811 INFO:tasks.rados.rados.0.smithi110.stdout:3108: done (2 left) 2024-09-14T03:48:21.811 INFO:tasks.rados.rados.0.smithi110.stdout:3109: done (1 left) 2024-09-14T03:48:21.811 INFO:tasks.rados.rados.0.smithi110.stdout:3110: done (0 left) 2024-09-14T03:48:21.811 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 308 2024-09-14T03:48:21.811 INFO:tasks.rados.rados.0.smithi110.stdout:3111: seq_num 955 ranges {0=802816,802816=557056,1359872=245760} 2024-09-14T03:48:21.815 INFO:tasks.rados.rados.0.smithi110.stdout:3111: writing smithi11075562-14 from 0 to 802816 tid 1 2024-09-14T03:48:21.818 INFO:tasks.rados.rados.0.smithi110.stdout:3111: writing smithi11075562-14 from 802816 to 1359872 tid 2 2024-09-14T03:48:21.820 INFO:tasks.rados.rados.0.smithi110.stdout:3111: writing smithi11075562-14 from 1359872 to 1605632 tid 3 2024-09-14T03:48:21.821 INFO:tasks.rados.rados.0.smithi110.stdout:3112: rollback oid 15 current snap is 308 2024-09-14T03:48:21.821 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 307 2024-09-14T03:48:21.821 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 308 2024-09-14T03:48:21.821 INFO:tasks.rados.rados.0.smithi110.stdout:3113: seq_num 956 ranges {1466368=417792,1884160=434176,2318336=770048} 2024-09-14T03:48:21.831 INFO:tasks.rados.rados.0.smithi110.stdout:3113: writing smithi11075562-46 from 1466368 to 1884160 tid 1 2024-09-14T03:48:21.833 INFO:tasks.rados.rados.0.smithi110.stdout:3113: writing smithi11075562-46 from 1884160 to 2318336 tid 2 2024-09-14T03:48:21.839 INFO:tasks.rados.rados.0.smithi110.stdout:3113: writing smithi11075562-46 from 2318336 to 3088384 tid 3 2024-09-14T03:48:21.840 INFO:tasks.rados.rados.0.smithi110.stdout:3114: read oid 49 snap 303 2024-09-14T03:48:21.840 INFO:tasks.rados.rados.0.smithi110.stdout:3114: expect (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:48:21.840 INFO:tasks.rados.rados.0.smithi110.stdout:3115: rmattr oid 8 current snap is 308 2024-09-14T03:48:21.840 INFO:tasks.rados.rados.0.smithi110.stdout:3115: done (4 left) 2024-09-14T03:48:21.840 INFO:tasks.rados.rados.0.smithi110.stdout:3116: snap_create 2024-09-14T03:48:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:3111: finishing write tid 1 to smithi11075562-14 2024-09-14T03:48:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:3111: finishing write tid 2 to smithi11075562-14 2024-09-14T03:48:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:3111: finishing write tid 3 to smithi11075562-14 2024-09-14T03:48:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:3111: finishing write tid 4 to smithi11075562-14 2024-09-14T03:48:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:3112: finishing rollback tid 0 to smithi11075562-15 2024-09-14T03:48:21.841 INFO:tasks.rados.rados.0.smithi110.stdout:3112: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:48:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:3112: finishing rollback tid 2 to smithi11075562-15 2024-09-14T03:48:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1994 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:48:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:3111: finishing write tid 5 to smithi11075562-14 2024-09-14T03:48:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1893 (ObjNum 955 snap 308 seq_num 955) dirty exists 2024-09-14T03:48:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:3111: left oid 14 (ObjNum 955 snap 308 seq_num 955) 2024-09-14T03:48:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:3113: finishing write tid 1 to smithi11075562-46 2024-09-14T03:48:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:3113: finishing write tid 2 to smithi11075562-46 2024-09-14T03:48:21.843 INFO:tasks.rados.rados.0.smithi110.stdout:3114: expect (ObjNum 877 snap 281 seq_num 877) 2024-09-14T03:48:21.850 INFO:tasks.rados.rados.0.smithi110.stdout:3113: finishing write tid 3 to smithi11075562-46 2024-09-14T03:48:21.850 INFO:tasks.rados.rados.0.smithi110.stdout:3113: finishing write tid 4 to smithi11075562-46 2024-09-14T03:48:21.850 INFO:tasks.rados.rados.0.smithi110.stdout:3113: finishing write tid 5 to smithi11075562-46 2024-09-14T03:48:21.850 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1995 (ObjNum 956 snap 308 seq_num 956) dirty exists 2024-09-14T03:48:21.850 INFO:tasks.rados.rados.0.smithi110.stdout:3113: left oid 46 (ObjNum 956 snap 308 seq_num 956) 2024-09-14T03:48:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:21 smithi110 ceph-mon[88262]: pgmap v673: 121 pgs: 121 active+clean; 314 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-14T03:48:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:21 smithi110 ceph-mon[88262]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T03:48:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:21 smithi110 ceph-mon[86423]: pgmap v673: 121 pgs: 121 active+clean; 314 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-14T03:48:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:21 smithi110 ceph-mon[86423]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T03:48:21.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:48:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:48:21] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-14T03:48:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:21 smithi195 ceph-mon[82388]: pgmap v673: 121 pgs: 121 active+clean; 314 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-14T03:48:22.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:21 smithi195 ceph-mon[82388]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T03:48:22.818 INFO:tasks.rados.rados.0.smithi110.stdout:3111: done (4 left) 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3112: done (3 left) 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3113: done (2 left) 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3114: done (1 left) 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3116: done (0 left) 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3117: read oid 5 snap -1 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3117: expect (ObjNum 936 snap 303 seq_num 936) 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3118: rmattr oid 35 current snap is 309 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3119: rollback oid 15 current snap is 309 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 308 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3120: copy_from oid 32 from oid 39 current snap is 309 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3121: read oid 47 snap -1 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3121: expect deleted 2024-09-14T03:48:22.819 INFO:tasks.rados.rados.0.smithi110.stdout:3122: setattr oid 6 current snap is 309 2024-09-14T03:48:22.820 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 309 2024-09-14T03:48:22.820 INFO:tasks.rados.rados.0.smithi110.stdout:3123: seq_num 957 ranges {1433600=434176,1867776=720896,2588672=483328} 2024-09-14T03:48:22.831 INFO:tasks.rados.rados.0.smithi110.stdout:3123: writing smithi11075562-36 from 1433600 to 1867776 tid 1 2024-09-14T03:48:22.836 INFO:tasks.rados.rados.0.smithi110.stdout:3123: writing smithi11075562-36 from 1867776 to 2588672 tid 2 2024-09-14T03:48:22.839 INFO:tasks.rados.rados.0.smithi110.stdout:3123: writing smithi11075562-36 from 2588672 to 3072000 tid 3 2024-09-14T03:48:22.839 INFO:tasks.rados.rados.0.smithi110.stdout:3124: read oid 45 snap -1 2024-09-14T03:48:22.839 INFO:tasks.rados.rados.0.smithi110.stdout:3124: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:48:22.839 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 309 2024-09-14T03:48:22.839 INFO:tasks.rados.rados.0.smithi110.stdout:3125: seq_num 958 ranges {0=450560,450560=655360,1105920=548864} 2024-09-14T03:48:22.841 INFO:tasks.rados.rados.0.smithi110.stdout:3125: writing smithi11075562-10 from 0 to 450560 tid 1 2024-09-14T03:48:22.845 INFO:tasks.rados.rados.0.smithi110.stdout:3125: writing smithi11075562-10 from 450560 to 1105920 tid 2 2024-09-14T03:48:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:3125: writing smithi11075562-10 from 1105920 to 1654784 tid 3 2024-09-14T03:48:22.848 INFO:tasks.rados.rados.0.smithi110.stdout:3126: delete oid 50 current snap is 309 2024-09-14T03:48:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:3119: finishing rollback tid 0 to smithi11075562-15 2024-09-14T03:48:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 2016 (ObjNum 33 snap 0 seq_num 65055856) dirty exists 2024-09-14T03:48:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2121 (ObjNum 953 snap 307 seq_num 953) dirty exists 2024-09-14T03:48:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:3119: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:48:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:3119: finishing rollback tid 2 to smithi11075562-15 2024-09-14T03:48:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1997 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:48:22.849 INFO:tasks.rados.rados.0.smithi110.stdout:3117: expect (ObjNum 936 snap 303 seq_num 936) 2024-09-14T03:48:23.050 INFO:tasks.rados.rados.0.smithi110.stdout:3123: finishing write tid 1 to smithi11075562-36 2024-09-14T03:48:23.050 INFO:tasks.rados.rados.0.smithi110.stdout:3123: finishing write tid 2 to smithi11075562-36 2024-09-14T03:48:23.050 INFO:tasks.rados.rados.0.smithi110.stdout:3123: finishing write tid 3 to smithi11075562-36 2024-09-14T03:48:23.050 INFO:tasks.rados.rados.0.smithi110.stdout:3123: finishing write tid 4 to smithi11075562-36 2024-09-14T03:48:23.050 INFO:tasks.rados.rados.0.smithi110.stdout:3123: finishing write tid 5 to smithi11075562-36 2024-09-14T03:48:23.051 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2190 (ObjNum 957 snap 309 seq_num 957) dirty exists 2024-09-14T03:48:23.051 INFO:tasks.rados.rados.0.smithi110.stdout:3123: left oid 36 (ObjNum 957 snap 309 seq_num 957) 2024-09-14T03:48:23.051 INFO:tasks.rados.rados.0.smithi110.stdout:3124: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3125: finishing write tid 1 to smithi11075562-10 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3125: finishing write tid 2 to smithi11075562-10 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3125: finishing write tid 3 to smithi11075562-10 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3125: finishing write tid 4 to smithi11075562-10 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3125: finishing write tid 5 to smithi11075562-10 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1892 (ObjNum 958 snap 309 seq_num 958) dirty exists 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3125: left oid 10 (ObjNum 958 snap 309 seq_num 958) 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3117: done (9 left) 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3118: done (8 left) 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3119: done (7 left) 2024-09-14T03:48:23.065 INFO:tasks.rados.rados.0.smithi110.stdout:3121: done (6 left) 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:3122: done (5 left) 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:3123: done (4 left) 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:3124: done (3 left) 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:3125: done (2 left) 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:3126: done (1 left) 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 309 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:3120: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:3120: finishing copy_from to smithi11075562-32 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 1833 (ObjNum 789 snap 259 seq_num 789) dirty exists 2024-09-14T03:48:23.066 INFO:tasks.rados.rados.0.smithi110.stdout:3127: seq_num 960 ranges {0=483328,483328=425984,909312=761856,1671168=16384} 2024-09-14T03:48:23.067 INFO:tasks.rados.rados.0.smithi110.stdout:3127: writing smithi11075562-47 from 0 to 483328 tid 1 2024-09-14T03:48:23.069 INFO:tasks.rados.rados.0.smithi110.stdout:3127: writing smithi11075562-47 from 483328 to 909312 tid 2 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3127: writing smithi11075562-47 from 909312 to 1671168 tid 3 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3127: writing smithi11075562-47 from 1671168 to 1687552 tid 4 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3120: done (1 left) 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3128: read oid 40 snap 308 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3128: expect deleted 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3127: finishing write tid 1 to smithi11075562-47 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3127: finishing write tid 2 to smithi11075562-47 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3129: copy_from oid 37 from oid 43 current snap is 309 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3130: delete oid 10 current snap is 309 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3128: done (3 left) 2024-09-14T03:48:23.077 INFO:tasks.rados.rados.0.smithi110.stdout:3130: done (2 left) 2024-09-14T03:48:23.078 INFO:tasks.rados.rados.0.smithi110.stdout:3131: snap_remove snap 303 2024-09-14T03:48:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:22 smithi110 ceph-mon[88262]: osdmap e697: 8 total, 8 up, 8 in 2024-09-14T03:48:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:22 smithi110 ceph-mon[86423]: osdmap e697: 8 total, 8 up, 8 in 2024-09-14T03:48:23.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:22 smithi195 ceph-mon[82388]: osdmap e697: 8 total, 8 up, 8 in 2024-09-14T03:48:23.789 INFO:tasks.rados.rados.0.smithi110.stdout:3131: done (2 left) 2024-09-14T03:48:23.789 INFO:tasks.rados.rados.0.smithi110.stdout:3132: read oid 13 snap 307 2024-09-14T03:48:23.789 INFO:tasks.rados.rados.0.smithi110.stdout:3132: expect (ObjNum 930 snap 302 seq_num 930) 2024-09-14T03:48:23.789 INFO:tasks.rados.rados.0.smithi110.stdout:3133: rollback oid 45 current snap is 309 2024-09-14T03:48:23.789 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 308 2024-09-14T03:48:23.789 INFO:tasks.rados.rados.0.smithi110.stdout:3127: finishing write tid 3 to smithi11075562-47 2024-09-14T03:48:23.789 INFO:tasks.rados.rados.0.smithi110.stdout:3127: finishing write tid 4 to smithi11075562-47 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:3127: finishing write tid 5 to smithi11075562-47 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:3127: finishing write tid 6 to smithi11075562-47 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 2195 (ObjNum 960 snap 309 seq_num 960) dirty exists 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:3127: left oid 47 (ObjNum 960 snap 309 seq_num 960) 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:3129: finishing copy_from racing read to smithi11075562-37 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:3129: finishing copy_from to smithi11075562-37 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 1824 (ObjNum 828 snap 274 seq_num 828) dirty exists 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:3127: done (3 left) 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:3129: done (2 left) 2024-09-14T03:48:23.790 INFO:tasks.rados.rados.0.smithi110.stdout:3134: snap_create 2024-09-14T03:48:23.808 INFO:tasks.rados.rados.0.smithi110.stdout:3132: expect (ObjNum 930 snap 302 seq_num 930) 2024-09-14T03:48:23.874 INFO:tasks.rados.rados.0.smithi110.stdout:3133: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:48:23.874 INFO:tasks.rados.rados.0.smithi110.stdout:3133: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:48:23.874 INFO:tasks.rados.rados.0.smithi110.stdout:3133: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:48:23.874 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2201 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-14T03:48:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:23 smithi110 ceph-mon[88262]: pgmap v676: 121 pgs: 121 active+clean; 305 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T03:48:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:23 smithi110 ceph-mon[88262]: osdmap e698: 8 total, 8 up, 8 in 2024-09-14T03:48:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:23 smithi110 ceph-mon[88262]: osdmap e699: 8 total, 8 up, 8 in 2024-09-14T03:48:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:23 smithi110 ceph-mon[86423]: pgmap v676: 121 pgs: 121 active+clean; 305 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T03:48:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:23 smithi110 ceph-mon[86423]: osdmap e698: 8 total, 8 up, 8 in 2024-09-14T03:48:24.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:23 smithi110 ceph-mon[86423]: osdmap e699: 8 total, 8 up, 8 in 2024-09-14T03:48:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:23 smithi195 ceph-mon[82388]: pgmap v676: 121 pgs: 121 active+clean; 305 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T03:48:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:23 smithi195 ceph-mon[82388]: osdmap e698: 8 total, 8 up, 8 in 2024-09-14T03:48:24.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:23 smithi195 ceph-mon[82388]: osdmap e699: 8 total, 8 up, 8 in 2024-09-14T03:48:24.789 INFO:tasks.rados.rados.0.smithi110.stdout:3132: done (2 left) 2024-09-14T03:48:24.789 INFO:tasks.rados.rados.0.smithi110.stdout:3133: done (1 left) 2024-09-14T03:48:24.790 INFO:tasks.rados.rados.0.smithi110.stdout:3134: done (0 left) 2024-09-14T03:48:24.790 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 30 current snap is 310 2024-09-14T03:48:24.790 INFO:tasks.rados.rados.0.smithi110.stdout:3135: seq_num 962 ranges {7544832=516096,8060928=589824,8650752=614400} 2024-09-14T03:48:24.832 INFO:tasks.rados.rados.0.smithi110.stdout:3135: writing smithi11075562-30 from 7544832 to 8060928 tid 1 2024-09-14T03:48:24.835 INFO:tasks.rados.rados.0.smithi110.stdout:3135: writing smithi11075562-30 from 8060928 to 8650752 tid 2 2024-09-14T03:48:24.839 INFO:tasks.rados.rados.0.smithi110.stdout:3135: writing smithi11075562-30 from 8650752 to 9265152 tid 3 2024-09-14T03:48:24.839 INFO:tasks.rados.rados.0.smithi110.stdout:3135: finishing write tid 1 to smithi11075562-30 2024-09-14T03:48:24.840 INFO:tasks.rados.rados.0.smithi110.stdout:3135: finishing write tid 2 to smithi11075562-30 2024-09-14T03:48:24.840 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 310 2024-09-14T03:48:24.840 INFO:tasks.rados.rados.0.smithi110.stdout:3136: seq_num 963 ranges {0=532480,532480=475136,1007616=532480,1540096=196608} 2024-09-14T03:48:24.842 INFO:tasks.rados.rados.0.smithi110.stdout:3136: writing smithi11075562-38 from 0 to 532480 tid 1 2024-09-14T03:48:24.845 INFO:tasks.rados.rados.0.smithi110.stdout:3136: writing smithi11075562-38 from 532480 to 1007616 tid 2 2024-09-14T03:48:24.848 INFO:tasks.rados.rados.0.smithi110.stdout:3136: writing smithi11075562-38 from 1007616 to 1540096 tid 3 2024-09-14T03:48:24.850 INFO:tasks.rados.rados.0.smithi110.stdout:3136: writing smithi11075562-38 from 1540096 to 1736704 tid 4 2024-09-14T03:48:24.850 INFO:tasks.rados.rados.0.smithi110.stdout:3137: snap_create 2024-09-14T03:48:24.850 INFO:tasks.rados.rados.0.smithi110.stdout:3135: finishing write tid 3 to smithi11075562-30 2024-09-14T03:48:24.850 INFO:tasks.rados.rados.0.smithi110.stdout:3135: finishing write tid 4 to smithi11075562-30 2024-09-14T03:48:24.851 INFO:tasks.rados.rados.0.smithi110.stdout:3135: finishing write tid 5 to smithi11075562-30 2024-09-14T03:48:24.851 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 2001 (ObjNum 962 snap 310 seq_num 962) dirty exists 2024-09-14T03:48:24.851 INFO:tasks.rados.rados.0.smithi110.stdout:3135: left oid 30 (ObjNum 962 snap 310 seq_num 962) 2024-09-14T03:48:24.851 INFO:tasks.rados.rados.0.smithi110.stdout:3136: finishing write tid 1 to smithi11075562-38 2024-09-14T03:48:24.851 INFO:tasks.rados.rados.0.smithi110.stdout:3136: finishing write tid 2 to smithi11075562-38 2024-09-14T03:48:24.852 INFO:tasks.rados.rados.0.smithi110.stdout:3136: finishing write tid 3 to smithi11075562-38 2024-09-14T03:48:24.852 INFO:tasks.rados.rados.0.smithi110.stdout:3136: finishing write tid 4 to smithi11075562-38 2024-09-14T03:48:24.852 INFO:tasks.rados.rados.0.smithi110.stdout:3136: finishing write tid 5 to smithi11075562-38 2024-09-14T03:48:24.853 INFO:tasks.rados.rados.0.smithi110.stdout:3136: finishing write tid 6 to smithi11075562-38 2024-09-14T03:48:24.853 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1622 (ObjNum 963 snap 310 seq_num 963) dirty exists 2024-09-14T03:48:24.853 INFO:tasks.rados.rados.0.smithi110.stdout:3136: left oid 38 (ObjNum 963 snap 310 seq_num 963) 2024-09-14T03:48:25.821 INFO:tasks.rados.rados.0.smithi110.stdout:3135: done (2 left) 2024-09-14T03:48:25.821 INFO:tasks.rados.rados.0.smithi110.stdout:3136: done (1 left) 2024-09-14T03:48:25.822 INFO:tasks.rados.rados.0.smithi110.stdout:3137: done (0 left) 2024-09-14T03:48:25.822 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 311 2024-09-14T03:48:25.822 INFO:tasks.rados.rados.0.smithi110.stdout:3138: seq_num 964 ranges {0=548864,548864=761856,1310720=442368} 2024-09-14T03:48:25.824 INFO:tasks.rados.rados.0.smithi110.stdout:3138: writing smithi11075562-31 from 0 to 548864 tid 1 2024-09-14T03:48:25.829 INFO:tasks.rados.rados.0.smithi110.stdout:3138: writing smithi11075562-31 from 548864 to 1310720 tid 2 2024-09-14T03:48:25.831 INFO:tasks.rados.rados.0.smithi110.stdout:3138: writing smithi11075562-31 from 1310720 to 1753088 tid 3 2024-09-14T03:48:25.831 INFO:tasks.rados.rados.0.smithi110.stdout:3139: rmattr oid 35 current snap is 311 2024-09-14T03:48:25.832 INFO:tasks.rados.rados.0.smithi110.stdout:3138: finishing write tid 1 to smithi11075562-31 2024-09-14T03:48:25.832 INFO:tasks.rados.rados.0.smithi110.stdout:3140: read oid 46 snap 307 2024-09-14T03:48:25.832 INFO:tasks.rados.rados.0.smithi110.stdout:3140: expect (ObjNum 907 snap 297 seq_num 907) 2024-09-14T03:48:25.832 INFO:tasks.rados.rados.0.smithi110.stdout:3141: copy_from oid 42 from oid 38 current snap is 311 2024-09-14T03:48:25.832 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 311 2024-09-14T03:48:25.832 INFO:tasks.rados.rados.0.smithi110.stdout:3142: seq_num 965 ranges {0=565248,565248=647168,1212416=548864,1761280=8192} 2024-09-14T03:48:25.834 INFO:tasks.rados.rados.0.smithi110.stdout:3142: writing smithi11075562-3 from 0 to 565248 tid 1 2024-09-14T03:48:25.840 INFO:tasks.rados.rados.0.smithi110.stdout:3142: writing smithi11075562-3 from 565248 to 1212416 tid 2 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3142: writing smithi11075562-3 from 1212416 to 1761280 tid 3 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3142: writing smithi11075562-3 from 1761280 to 1769472 tid 4 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3143: delete oid 40 current snap is 311 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3138: finishing write tid 2 to smithi11075562-31 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3138: finishing write tid 3 to smithi11075562-31 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3138: finishing write tid 4 to smithi11075562-31 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 2017 (ObjNum 545 snap 0 seq_num 65178128) dirty exists 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3138: finishing write tid 5 to smithi11075562-31 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 2123 (ObjNum 964 snap 311 seq_num 964) dirty exists 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3138: left oid 31 (ObjNum 964 snap 311 seq_num 964) 2024-09-14T03:48:25.845 INFO:tasks.rados.rados.0.smithi110.stdout:3140: expect (ObjNum 907 snap 297 seq_num 907) 2024-09-14T03:48:25.897 INFO:tasks.rados.rados.0.smithi110.stdout:3142: finishing write tid 1 to smithi11075562-3 2024-09-14T03:48:25.897 INFO:tasks.rados.rados.0.smithi110.stdout:3142: finishing write tid 2 to smithi11075562-3 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:3138: done (5 left) 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:3139: done (4 left) 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:3140: done (3 left) 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:3143: done (2 left) 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:3144: read oid 4 snap -1 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:3144: expect (ObjNum 944 snap 305 seq_num 944) 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 311 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:3141: finishing copy_from racing read to smithi11075562-42 2024-09-14T03:48:25.898 INFO:tasks.rados.rados.0.smithi110.stdout:3145: seq_num 967 ranges {5234688=598016,5832704=466944,6299648=516096,6815744=221184} 2024-09-14T03:48:25.929 INFO:tasks.rados.rados.0.smithi110.stdout:3145: writing smithi11075562-23 from 5234688 to 5832704 tid 1 2024-09-14T03:48:25.931 INFO:tasks.rados.rados.0.smithi110.stdout:3145: writing smithi11075562-23 from 5832704 to 6299648 tid 2 2024-09-14T03:48:25.934 INFO:tasks.rados.rados.0.smithi110.stdout:3145: writing smithi11075562-23 from 6299648 to 6815744 tid 3 2024-09-14T03:48:25.936 INFO:tasks.rados.rados.0.smithi110.stdout:3145: writing smithi11075562-23 from 6815744 to 7036928 tid 4 2024-09-14T03:48:25.936 INFO:tasks.rados.rados.0.smithi110.stdout:3146: read oid 24 snap -1 2024-09-14T03:48:25.936 INFO:tasks.rados.rados.0.smithi110.stdout:3146: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:48:25.936 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 311 2024-09-14T03:48:25.936 INFO:tasks.rados.rados.0.smithi110.stdout:3142: finishing write tid 3 to smithi11075562-3 2024-09-14T03:48:25.936 INFO:tasks.rados.rados.0.smithi110.stdout:3142: finishing write tid 4 to smithi11075562-3 2024-09-14T03:48:25.936 INFO:tasks.rados.rados.0.smithi110.stdout:3147: seq_num 968 ranges {2048000=614400,2662400=745472,3407872=458752} 2024-09-14T03:48:25.950 INFO:tasks.rados.rados.0.smithi110.stdout:3147: writing smithi11075562-7 from 2048000 to 2662400 tid 1 2024-09-14T03:48:25.955 INFO:tasks.rados.rados.0.smithi110.stdout:3147: writing smithi11075562-7 from 2662400 to 3407872 tid 2 2024-09-14T03:48:25.958 INFO:tasks.rados.rados.0.smithi110.stdout:3147: writing smithi11075562-7 from 3407872 to 3866624 tid 3 2024-09-14T03:48:25.958 INFO:tasks.rados.rados.0.smithi110.stdout:3148: copy_from oid 10 from oid 33 current snap is 311 2024-09-14T03:48:25.958 INFO:tasks.rados.rados.0.smithi110.stdout:3142: finishing write tid 5 to smithi11075562-3 2024-09-14T03:48:25.958 INFO:tasks.rados.rados.0.smithi110.stdout:3141: finishing copy_from to smithi11075562-42 2024-09-14T03:48:25.958 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1622 (ObjNum 963 snap 310 seq_num 963) dirty exists 2024-09-14T03:48:25.959 INFO:tasks.rados.rados.0.smithi110.stdout:3142: finishing write tid 6 to smithi11075562-3 2024-09-14T03:48:25.959 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 1683 (ObjNum 965 snap 311 seq_num 965) dirty exists 2024-09-14T03:48:25.959 INFO:tasks.rados.rados.0.smithi110.stdout:3142: left oid 3 (ObjNum 965 snap 311 seq_num 965) 2024-09-14T03:48:25.959 INFO:tasks.rados.rados.0.smithi110.stdout:3144: expect (ObjNum 944 snap 305 seq_num 944) 2024-09-14T03:48:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:25 smithi110 ceph-mon[88262]: pgmap v679: 121 pgs: 121 active+clean; 291 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-14T03:48:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:25 smithi110 ceph-mon[88262]: osdmap e700: 8 total, 8 up, 8 in 2024-09-14T03:48:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:25 smithi110 ceph-mon[86423]: pgmap v679: 121 pgs: 121 active+clean; 291 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-14T03:48:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:25 smithi110 ceph-mon[86423]: osdmap e700: 8 total, 8 up, 8 in 2024-09-14T03:48:26.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:25 smithi195 ceph-mon[82388]: pgmap v679: 121 pgs: 121 active+clean; 291 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-14T03:48:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:25 smithi195 ceph-mon[82388]: osdmap e700: 8 total, 8 up, 8 in 2024-09-14T03:48:26.265 INFO:tasks.rados.rados.0.smithi110.stdout:3145: finishing write tid 1 to smithi11075562-23 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3145: finishing write tid 2 to smithi11075562-23 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3145: finishing write tid 3 to smithi11075562-23 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3145: finishing write tid 4 to smithi11075562-23 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3145: finishing write tid 5 to smithi11075562-23 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3147: finishing write tid 1 to smithi11075562-7 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3145: finishing write tid 6 to smithi11075562-23 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 1898 (ObjNum 967 snap 311 seq_num 967) dirty exists 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3145: left oid 23 (ObjNum 967 snap 311 seq_num 967) 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3141: done (6 left) 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3142: done (5 left) 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3144: done (4 left) 2024-09-14T03:48:26.266 INFO:tasks.rados.rados.0.smithi110.stdout:3145: done (3 left) 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:3149: rmattr oid 23 current snap is 311 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:3149: done (3 left) 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 311 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:3147: finishing write tid 2 to smithi11075562-7 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:3147: finishing write tid 3 to smithi11075562-7 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:3147: finishing write tid 4 to smithi11075562-7 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:3147: finishing write tid 5 to smithi11075562-7 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1626 (ObjNum 968 snap 311 seq_num 968) dirty exists 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:3147: left oid 7 (ObjNum 968 snap 311 seq_num 968) 2024-09-14T03:48:26.267 INFO:tasks.rados.rados.0.smithi110.stdout:3146: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:48:26.823 INFO:tasks.rados.rados.0.smithi110.stdout:3148: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:48:26.824 INFO:tasks.rados.rados.0.smithi110.stdout:3148: finishing copy_from to smithi11075562-10 2024-09-14T03:48:26.824 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1990 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:48:26.824 INFO:tasks.rados.rados.0.smithi110.stdout:3150: seq_num 969 ranges {1384448=638976,2023424=630784,2654208=532480,3186688=40960} 2024-09-14T03:48:26.834 INFO:tasks.rados.rados.0.smithi110.stdout:3150: writing smithi11075562-28 from 1384448 to 2023424 tid 1 2024-09-14T03:48:26.838 INFO:tasks.rados.rados.0.smithi110.stdout:3150: writing smithi11075562-28 from 2023424 to 2654208 tid 2 2024-09-14T03:48:26.841 INFO:tasks.rados.rados.0.smithi110.stdout:3150: writing smithi11075562-28 from 2654208 to 3186688 tid 3 2024-09-14T03:48:26.842 INFO:tasks.rados.rados.0.smithi110.stdout:3150: writing smithi11075562-28 from 3186688 to 3227648 tid 4 2024-09-14T03:48:26.842 INFO:tasks.rados.rados.0.smithi110.stdout:3146: done (3 left) 2024-09-14T03:48:26.842 INFO:tasks.rados.rados.0.smithi110.stdout:3147: done (2 left) 2024-09-14T03:48:26.842 INFO:tasks.rados.rados.0.smithi110.stdout:3148: done (1 left) 2024-09-14T03:48:26.842 INFO:tasks.rados.rados.0.smithi110.stdout:3151: snap_create 2024-09-14T03:48:26.842 INFO:tasks.rados.rados.0.smithi110.stdout:3150: finishing write tid 1 to smithi11075562-28 2024-09-14T03:48:26.842 INFO:tasks.rados.rados.0.smithi110.stdout:3150: finishing write tid 2 to smithi11075562-28 2024-09-14T03:48:26.844 INFO:tasks.rados.rados.0.smithi110.stdout:3150: finishing write tid 3 to smithi11075562-28 2024-09-14T03:48:26.844 INFO:tasks.rados.rados.0.smithi110.stdout:3150: finishing write tid 4 to smithi11075562-28 2024-09-14T03:48:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:3150: finishing write tid 5 to smithi11075562-28 2024-09-14T03:48:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:3150: finishing write tid 6 to smithi11075562-28 2024-09-14T03:48:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 1688 (ObjNum 969 snap 311 seq_num 969) dirty exists 2024-09-14T03:48:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:3150: left oid 28 (ObjNum 969 snap 311 seq_num 969) 2024-09-14T03:48:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:26 smithi110 ceph-mon[88262]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T03:48:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:26 smithi110 ceph-mon[88262]: pgmap v682: 121 pgs: 121 active+clean; 275 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 6.8 MiB/s wr, 20 op/s 2024-09-14T03:48:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:26 smithi110 ceph-mon[86423]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T03:48:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:26 smithi110 ceph-mon[86423]: pgmap v682: 121 pgs: 121 active+clean; 275 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 6.8 MiB/s wr, 20 op/s 2024-09-14T03:48:27.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:26 smithi195 ceph-mon[82388]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T03:48:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:26 smithi195 ceph-mon[82388]: pgmap v682: 121 pgs: 121 active+clean; 275 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 6.8 MiB/s wr, 20 op/s 2024-09-14T03:48:27.831 INFO:tasks.rados.rados.0.smithi110.stdout:3150: done (1 left) 2024-09-14T03:48:27.831 INFO:tasks.rados.rados.0.smithi110.stdout:3151: done (0 left) 2024-09-14T03:48:27.831 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 312 2024-09-14T03:48:27.831 INFO:tasks.rados.rados.0.smithi110.stdout:3152: seq_num 970 ranges {15400960=655360,16056320=516096,16572416=688128} 2024-09-14T03:48:27.915 INFO:tasks.rados.rados.0.smithi110.stdout:3152: writing smithi11075562-39 from 15400960 to 16056320 tid 1 2024-09-14T03:48:27.918 INFO:tasks.rados.rados.0.smithi110.stdout:3152: writing smithi11075562-39 from 16056320 to 16572416 tid 2 2024-09-14T03:48:27.922 INFO:tasks.rados.rados.0.smithi110.stdout:3152: writing smithi11075562-39 from 16572416 to 17260544 tid 3 2024-09-14T03:48:27.922 INFO:tasks.rados.rados.0.smithi110.stdout:3153: read oid 17 snap -1 2024-09-14T03:48:27.922 INFO:tasks.rados.rados.0.smithi110.stdout:3153: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:27.922 INFO:tasks.rados.rados.0.smithi110.stdout:3154: snap_create 2024-09-14T03:48:27.922 INFO:tasks.rados.rados.0.smithi110.stdout:3152: finishing write tid 1 to smithi11075562-39 2024-09-14T03:48:27.922 INFO:tasks.rados.rados.0.smithi110.stdout:3152: finishing write tid 2 to smithi11075562-39 2024-09-14T03:48:27.926 INFO:tasks.rados.rados.0.smithi110.stdout:3152: finishing write tid 3 to smithi11075562-39 2024-09-14T03:48:27.926 INFO:tasks.rados.rados.0.smithi110.stdout:3152: finishing write tid 4 to smithi11075562-39 2024-09-14T03:48:27.927 INFO:tasks.rados.rados.0.smithi110.stdout:3152: finishing write tid 5 to smithi11075562-39 2024-09-14T03:48:27.927 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1897 (ObjNum 970 snap 312 seq_num 970) dirty exists 2024-09-14T03:48:27.927 INFO:tasks.rados.rados.0.smithi110.stdout:3152: left oid 39 (ObjNum 970 snap 312 seq_num 970) 2024-09-14T03:48:27.944 INFO:tasks.rados.rados.0.smithi110.stdout:3153: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:28.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:28.787 INFO:tasks.rados.rados.0.smithi110.stdout:3152: done (2 left) 2024-09-14T03:48:28.787 INFO:tasks.rados.rados.0.smithi110.stdout:3153: done (1 left) 2024-09-14T03:48:28.787 INFO:tasks.rados.rados.0.smithi110.stdout:3154: done (0 left) 2024-09-14T03:48:28.788 INFO:tasks.rados.rados.0.smithi110.stdout:3155: snap_create 2024-09-14T03:48:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:28 smithi110 ceph-mon[88262]: pgmap v683: 121 pgs: 121 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.1 MiB/s wr, 22 op/s 2024-09-14T03:48:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:28 smithi110 ceph-mon[88262]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T03:48:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:28 smithi110 ceph-mon[88262]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T03:48:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:28 smithi110 ceph-mon[86423]: pgmap v683: 121 pgs: 121 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.1 MiB/s wr, 22 op/s 2024-09-14T03:48:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:28 smithi110 ceph-mon[86423]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T03:48:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:28 smithi110 ceph-mon[86423]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T03:48:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:28 smithi195 ceph-mon[82388]: pgmap v683: 121 pgs: 121 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.1 MiB/s wr, 22 op/s 2024-09-14T03:48:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:28 smithi195 ceph-mon[82388]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T03:48:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:28 smithi195 ceph-mon[82388]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T03:48:29.790 INFO:tasks.rados.rados.0.smithi110.stdout:3155: done (0 left) 2024-09-14T03:48:29.790 INFO:tasks.rados.rados.0.smithi110.stdout:3156: rollback oid 32 current snap is 314 2024-09-14T03:48:29.790 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 279 2024-09-14T03:48:29.791 INFO:tasks.rados.rados.0.smithi110.stdout:3157: copy_from oid 1 from oid 15 current snap is 314 2024-09-14T03:48:29.791 INFO:tasks.rados.rados.0.smithi110.stdout:3158: snap_remove snap 313 2024-09-14T03:48:30.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:48:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:48:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:48:30.797 INFO:tasks.rados.rados.0.smithi110.stdout:3158: done (2 left) 2024-09-14T03:48:30.797 INFO:tasks.rados.rados.0.smithi110.stdout:3159: copy_from oid 40 from oid 30 current snap is 314 2024-09-14T03:48:30.797 INFO:tasks.rados.rados.0.smithi110.stdout:3160: read oid 29 snap -1 2024-09-14T03:48:30.797 INFO:tasks.rados.rados.0.smithi110.stdout:3160: expect deleted 2024-09-14T03:48:30.798 INFO:tasks.rados.rados.0.smithi110.stdout:3161: read oid 6 snap -1 2024-09-14T03:48:30.798 INFO:tasks.rados.rados.0.smithi110.stdout:3161: expect (ObjNum 953 snap 307 seq_num 953) 2024-09-14T03:48:30.798 INFO:tasks.rados.rados.0.smithi110.stdout:3162: setattr oid 24 current snap is 314 2024-09-14T03:48:30.798 INFO:tasks.rados.rados.0.smithi110.stdout:3156: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:48:30.798 INFO:tasks.rados.rados.0.smithi110.stdout:3156: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:48:30.798 INFO:tasks.rados.rados.0.smithi110.stdout:3156: finishing rollback tid 2 to smithi11075562-32 2024-09-14T03:48:30.798 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2004 (ObjNum 789 snap 259 seq_num 789) dirty exists 2024-09-14T03:48:30.799 INFO:tasks.rados.rados.0.smithi110.stdout:3157: finishing copy_from racing read to smithi11075562-1 2024-09-14T03:48:30.799 INFO:tasks.rados.rados.0.smithi110.stdout:3157: finishing copy_from to smithi11075562-1 2024-09-14T03:48:30.799 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 1997 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:48:30.799 INFO:tasks.rados.rados.0.smithi110.stdout:3156: done (5 left) 2024-09-14T03:48:30.800 INFO:tasks.rados.rados.0.smithi110.stdout:3157: done (4 left) 2024-09-14T03:48:30.800 INFO:tasks.rados.rados.0.smithi110.stdout:3160: done (3 left) 2024-09-14T03:48:30.800 INFO:tasks.rados.rados.0.smithi110.stdout:3163: read oid 47 snap -1 2024-09-14T03:48:30.800 INFO:tasks.rados.rados.0.smithi110.stdout:3163: expect (ObjNum 960 snap 309 seq_num 960) 2024-09-14T03:48:30.800 INFO:tasks.rados.rados.0.smithi110.stdout:3164: setattr oid 26 current snap is 314 2024-09-14T03:48:30.801 INFO:tasks.rados.rados.0.smithi110.stdout:3165: delete oid 16 current snap is 314 2024-09-14T03:48:30.807 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 1991 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:48:30.812 INFO:tasks.rados.rados.0.smithi110.stdout:3163: expect (ObjNum 960 snap 309 seq_num 960) 2024-09-14T03:48:30.865 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1837 (ObjNum 948 snap 307 seq_num 948) dirty exists 2024-09-14T03:48:30.866 INFO:tasks.rados.rados.0.smithi110.stdout:3161: expect (ObjNum 953 snap 307 seq_num 953) 2024-09-14T03:48:31.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:30 smithi110 ceph-mon[88262]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T03:48:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:30 smithi110 ceph-mon[86423]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T03:48:31.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:30 smithi195 ceph-mon[82388]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T03:48:31.400 INFO:tasks.rados.rados.0.smithi110.stdout:3159: finishing copy_from racing read to smithi11075562-40 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:3159: finishing copy_from to smithi11075562-40 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 2001 (ObjNum 962 snap 310 seq_num 962) dirty exists 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:3159: done (5 left) 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:3161: done (4 left) 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:3162: done (3 left) 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:3163: done (2 left) 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:3164: done (1 left) 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:3165: done (0 left) 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 314 2024-09-14T03:48:31.401 INFO:tasks.rados.rados.0.smithi110.stdout:3166: seq_num 972 ranges {0=688128,688128=688128,1376256=516096} 2024-09-14T03:48:31.404 INFO:tasks.rados.rados.0.smithi110.stdout:3166: writing smithi11075562-50 from 0 to 688128 tid 1 2024-09-14T03:48:31.408 INFO:tasks.rados.rados.0.smithi110.stdout:3166: writing smithi11075562-50 from 688128 to 1376256 tid 2 2024-09-14T03:48:31.411 INFO:tasks.rados.rados.0.smithi110.stdout:3166: writing smithi11075562-50 from 1376256 to 1892352 tid 3 2024-09-14T03:48:31.411 INFO:tasks.rados.rados.0.smithi110.stdout:3167: delete oid 32 current snap is 314 2024-09-14T03:48:31.411 INFO:tasks.rados.rados.0.smithi110.stdout:3166: finishing write tid 1 to smithi11075562-50 2024-09-14T03:48:31.412 INFO:tasks.rados.rados.0.smithi110.stdout:3166: finishing write tid 2 to smithi11075562-50 2024-09-14T03:48:31.414 INFO:tasks.rados.rados.0.smithi110.stdout:3166: finishing write tid 3 to smithi11075562-50 2024-09-14T03:48:31.414 INFO:tasks.rados.rados.0.smithi110.stdout:3167: done (1 left) 2024-09-14T03:48:31.414 INFO:tasks.rados.rados.0.smithi110.stdout:3168: rollback oid 27 current snap is 314 2024-09-14T03:48:31.414 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 308 2024-09-14T03:48:31.414 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 314 2024-09-14T03:48:31.414 INFO:tasks.rados.rados.0.smithi110.stdout:3169: seq_num 974 ranges {11673600=720896,12394496=507904,12902400=671744,13574144=24576} 2024-09-14T03:48:31.479 INFO:tasks.rados.rados.0.smithi110.stdout:3169: writing smithi11075562-26 from 11673600 to 12394496 tid 1 2024-09-14T03:48:31.481 INFO:tasks.rados.rados.0.smithi110.stdout:3169: writing smithi11075562-26 from 12394496 to 12902400 tid 2 2024-09-14T03:48:31.485 INFO:tasks.rados.rados.0.smithi110.stdout:3169: writing smithi11075562-26 from 12902400 to 13574144 tid 3 2024-09-14T03:48:31.486 INFO:tasks.rados.rados.0.smithi110.stdout:3169: writing smithi11075562-26 from 13574144 to 13598720 tid 4 2024-09-14T03:48:31.486 INFO:tasks.rados.rados.0.smithi110.stdout:3166: finishing write tid 4 to smithi11075562-50 2024-09-14T03:48:31.486 INFO:tasks.rados.rados.0.smithi110.stdout:3168: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:48:31.486 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-14T03:48:31.486 INFO:tasks.rados.rados.0.smithi110.stdout:3166: finishing write tid 5 to smithi11075562-50 2024-09-14T03:48:31.486 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 1510 (ObjNum 972 snap 314 seq_num 972) dirty exists 2024-09-14T03:48:31.486 INFO:tasks.rados.rados.0.smithi110.stdout:3166: left oid 50 (ObjNum 972 snap 314 seq_num 972) 2024-09-14T03:48:31.486 INFO:tasks.rados.rados.0.smithi110.stdout:3166: done (2 left) 2024-09-14T03:48:31.487 INFO:tasks.rados.rados.0.smithi110.stdout:3168: done (1 left) 2024-09-14T03:48:31.487 INFO:tasks.rados.rados.0.smithi110.stdout:3170: read oid 11 snap -1 2024-09-14T03:48:31.487 INFO:tasks.rados.rados.0.smithi110.stdout:3170: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:31.487 INFO:tasks.rados.rados.0.smithi110.stdout:3171: snap_remove snap 312 2024-09-14T03:48:31.799 INFO:tasks.rados.rados.0.smithi110.stdout:3171: done (2 left) 2024-09-14T03:48:31.799 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 314 2024-09-14T03:48:31.801 INFO:tasks.rados.rados.0.smithi110.stdout:3169: finishing write tid 1 to smithi11075562-26 2024-09-14T03:48:31.802 INFO:tasks.rados.rados.0.smithi110.stdout:3169: finishing write tid 2 to smithi11075562-26 2024-09-14T03:48:31.802 INFO:tasks.rados.rados.0.smithi110.stdout:3169: finishing write tid 3 to smithi11075562-26 2024-09-14T03:48:31.802 INFO:tasks.rados.rados.0.smithi110.stdout:3169: finishing write tid 4 to smithi11075562-26 2024-09-14T03:48:31.802 INFO:tasks.rados.rados.0.smithi110.stdout:3169: finishing write tid 5 to smithi11075562-26 2024-09-14T03:48:31.802 INFO:tasks.rados.rados.0.smithi110.stdout:3169: finishing write tid 6 to smithi11075562-26 2024-09-14T03:48:31.802 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1842 (ObjNum 974 snap 314 seq_num 974) dirty exists 2024-09-14T03:48:31.802 INFO:tasks.rados.rados.0.smithi110.stdout:3169: left oid 26 (ObjNum 974 snap 314 seq_num 974) 2024-09-14T03:48:31.802 INFO:tasks.rados.rados.0.smithi110.stdout:3170: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:31.872 INFO:tasks.rados.rados.0.smithi110.stdout:3172: seq_num 975 ranges {3072000=737280,3809280=786432,4595712=417792} 2024-09-14T03:48:31.892 INFO:tasks.rados.rados.0.smithi110.stdout:3172: writing smithi11075562-36 from 3072000 to 3809280 tid 1 2024-09-14T03:48:31.897 INFO:tasks.rados.rados.0.smithi110.stdout:3172: writing smithi11075562-36 from 3809280 to 4595712 tid 2 2024-09-14T03:48:31.899 INFO:tasks.rados.rados.0.smithi110.stdout:3172: writing smithi11075562-36 from 4595712 to 5013504 tid 3 2024-09-14T03:48:31.899 INFO:tasks.rados.rados.0.smithi110.stdout:3169: done (2 left) 2024-09-14T03:48:31.899 INFO:tasks.rados.rados.0.smithi110.stdout:3170: done (1 left) 2024-09-14T03:48:31.899 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 314 2024-09-14T03:48:31.899 INFO:tasks.rados.rados.0.smithi110.stdout:3173: seq_num 976 ranges {10485760=753664,11239424=671744,11911168=532480} 2024-09-14T03:48:31.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:31 smithi110 ceph-mon[88262]: pgmap v687: 121 pgs: 121 active+clean; 300 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:48:31.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:31 smithi110 ceph-mon[88262]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T03:48:31.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:48:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:48:31] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-14T03:48:31.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:31 smithi110 ceph-mon[86423]: pgmap v687: 121 pgs: 121 active+clean; 300 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:48:31.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:31 smithi110 ceph-mon[86423]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T03:48:31.958 INFO:tasks.rados.rados.0.smithi110.stdout:3173: writing smithi11075562-44 from 10485760 to 11239424 tid 1 2024-09-14T03:48:31.962 INFO:tasks.rados.rados.0.smithi110.stdout:3173: writing smithi11075562-44 from 11239424 to 11911168 tid 2 2024-09-14T03:48:31.965 INFO:tasks.rados.rados.0.smithi110.stdout:3173: writing smithi11075562-44 from 11911168 to 12443648 tid 3 2024-09-14T03:48:31.965 INFO:tasks.rados.rados.0.smithi110.stdout:3174: snap_remove snap 308 2024-09-14T03:48:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:31 smithi195 ceph-mon[82388]: pgmap v687: 121 pgs: 121 active+clean; 300 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:48:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:31 smithi195 ceph-mon[82388]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3174: done (2 left) 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3172: finishing write tid 1 to smithi11075562-36 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3172: finishing write tid 2 to smithi11075562-36 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3172: finishing write tid 3 to smithi11075562-36 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3172: finishing write tid 4 to smithi11075562-36 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3172: finishing write tid 5 to smithi11075562-36 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2216 (ObjNum 975 snap 314 seq_num 975) dirty exists 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3172: left oid 36 (ObjNum 975 snap 314 seq_num 975) 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3173: finishing write tid 1 to smithi11075562-44 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3173: finishing write tid 2 to smithi11075562-44 2024-09-14T03:48:32.806 INFO:tasks.rados.rados.0.smithi110.stdout:3173: finishing write tid 3 to smithi11075562-44 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:3173: finishing write tid 4 to smithi11075562-44 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:3173: finishing write tid 5 to smithi11075562-44 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 1822 (ObjNum 976 snap 314 seq_num 976) dirty exists 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:3173: left oid 44 (ObjNum 976 snap 314 seq_num 976) 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:3175: read oid 43 snap -1 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:3175: expect (ObjNum 828 snap 274 seq_num 828) 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:3172: done (2 left) 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:3173: done (1 left) 2024-09-14T03:48:32.807 INFO:tasks.rados.rados.0.smithi110.stdout:3176: setattr oid 45 current snap is 314 2024-09-14T03:48:32.808 INFO:tasks.rados.rados.0.smithi110.stdout:3177: delete oid 47 current snap is 314 2024-09-14T03:48:32.823 INFO:tasks.rados.rados.0.smithi110.stdout:3175: expect (ObjNum 828 snap 274 seq_num 828) 2024-09-14T03:48:32.938 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2218 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-14T03:48:32.938 INFO:tasks.rados.rados.0.smithi110.stdout:3175: done (2 left) 2024-09-14T03:48:32.939 INFO:tasks.rados.rados.0.smithi110.stdout:3176: done (1 left) 2024-09-14T03:48:32.939 INFO:tasks.rados.rados.0.smithi110.stdout:3177: done (0 left) 2024-09-14T03:48:32.939 INFO:tasks.rados.rados.0.smithi110.stdout:3178: delete oid 8 current snap is 314 2024-09-14T03:48:32.941 INFO:tasks.rados.rados.0.smithi110.stdout:3178: done (0 left) 2024-09-14T03:48:32.941 INFO:tasks.rados.rados.0.smithi110.stdout:3179: snap_create 2024-09-14T03:48:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:32 smithi110 ceph-mon[88262]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T03:48:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:32 smithi110 ceph-mon[86423]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T03:48:33.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:32 smithi195 ceph-mon[82388]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T03:48:33.790 INFO:tasks.rados.rados.0.smithi110.stdout:3179: done (0 left) 2024-09-14T03:48:33.790 INFO:tasks.rados.rados.0.smithi110.stdout:3180: copy_from oid 49 from oid 13 current snap is 315 2024-09-14T03:48:33.791 INFO:tasks.rados.rados.0.smithi110.stdout:3181: copy_from oid 46 from oid 17 current snap is 315 2024-09-14T03:48:33.791 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 315 2024-09-14T03:48:33.791 INFO:tasks.rados.rados.0.smithi110.stdout:3182: seq_num 979 ranges {1146880=802816,1949696=729088,2678784=458752,3137536=16384} 2024-09-14T03:48:33.801 INFO:tasks.rados.rados.0.smithi110.stdout:3182: writing smithi11075562-25 from 1146880 to 1949696 tid 1 2024-09-14T03:48:33.806 INFO:tasks.rados.rados.0.smithi110.stdout:3182: writing smithi11075562-25 from 1949696 to 2678784 tid 2 2024-09-14T03:48:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3182: writing smithi11075562-25 from 2678784 to 3137536 tid 3 2024-09-14T03:48:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3182: writing smithi11075562-25 from 3137536 to 3153920 tid 4 2024-09-14T03:48:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3183: read oid 11 snap -1 2024-09-14T03:48:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3183: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 315 2024-09-14T03:48:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3180: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:48:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3184: seq_num 980 ranges {3391488=417792,3809280=614400,4423680=663552,5087232=327680} 2024-09-14T03:48:33.831 INFO:tasks.rados.rados.0.smithi110.stdout:3184: writing smithi11075562-12 from 3391488 to 3809280 tid 1 2024-09-14T03:48:33.834 INFO:tasks.rados.rados.0.smithi110.stdout:3184: writing smithi11075562-12 from 3809280 to 4423680 tid 2 2024-09-14T03:48:33.838 INFO:tasks.rados.rados.0.smithi110.stdout:3184: writing smithi11075562-12 from 4423680 to 5087232 tid 3 2024-09-14T03:48:33.840 INFO:tasks.rados.rados.0.smithi110.stdout:3184: writing smithi11075562-12 from 5087232 to 5414912 tid 4 2024-09-14T03:48:33.840 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 315 2024-09-14T03:48:33.840 INFO:tasks.rados.rados.0.smithi110.stdout:3180: finishing copy_from to smithi11075562-49 2024-09-14T03:48:33.840 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 1998 (ObjNum 930 snap 302 seq_num 930) dirty exists 2024-09-14T03:48:33.840 INFO:tasks.rados.rados.0.smithi110.stdout:3182: finishing write tid 1 to smithi11075562-25 2024-09-14T03:48:33.841 INFO:tasks.rados.rados.0.smithi110.stdout:3185: seq_num 981 ranges {0=434176,434176=540672,974848=466944,1441792=524288,1966080=73728} 2024-09-14T03:48:33.842 INFO:tasks.rados.rados.0.smithi110.stdout:3185: writing smithi11075562-32 from 0 to 434176 tid 1 2024-09-14T03:48:33.846 INFO:tasks.rados.rados.0.smithi110.stdout:3185: writing smithi11075562-32 from 434176 to 974848 tid 2 2024-09-14T03:48:33.849 INFO:tasks.rados.rados.0.smithi110.stdout:3185: writing smithi11075562-32 from 974848 to 1441792 tid 3 2024-09-14T03:48:33.852 INFO:tasks.rados.rados.0.smithi110.stdout:3185: writing smithi11075562-32 from 1441792 to 1966080 tid 4 2024-09-14T03:48:33.854 INFO:tasks.rados.rados.0.smithi110.stdout:3185: writing smithi11075562-32 from 1966080 to 2039808 tid 5 2024-09-14T03:48:33.854 INFO:tasks.rados.rados.0.smithi110.stdout:3180: done (5 left) 2024-09-14T03:48:33.854 INFO:tasks.rados.rados.0.smithi110.stdout:3186: rollback oid 30 current snap is 315 2024-09-14T03:48:33.854 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 311 2024-09-14T03:48:33.854 INFO:tasks.rados.rados.0.smithi110.stdout:3181: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:48:33.854 INFO:tasks.rados.rados.0.smithi110.stdout:3182: finishing write tid 2 to smithi11075562-25 2024-09-14T03:48:33.854 INFO:tasks.rados.rados.0.smithi110.stdout:3187: delete oid 35 current snap is 315 2024-09-14T03:48:33.854 INFO:tasks.rados.rados.0.smithi110.stdout:3183: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:33.925 INFO:tasks.rados.rados.0.smithi110.stdout:3182: finishing write tid 3 to smithi11075562-25 2024-09-14T03:48:33.925 INFO:tasks.rados.rados.0.smithi110.stdout:3182: finishing write tid 4 to smithi11075562-25 2024-09-14T03:48:33.925 INFO:tasks.rados.rados.0.smithi110.stdout:3182: finishing write tid 5 to smithi11075562-25 2024-09-14T03:48:33.925 INFO:tasks.rados.rados.0.smithi110.stdout:3181: finishing copy_from to smithi11075562-46 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 1885 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3182: finishing write tid 6 to smithi11075562-25 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2232 (ObjNum 979 snap 315 seq_num 979) dirty exists 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3182: left oid 25 (ObjNum 979 snap 315 seq_num 979) 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3184: finishing write tid 1 to smithi11075562-12 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3184: finishing write tid 2 to smithi11075562-12 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3184: finishing write tid 3 to smithi11075562-12 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3184: finishing write tid 4 to smithi11075562-12 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3184: finishing write tid 5 to smithi11075562-12 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3181: done (6 left) 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3182: done (5 left) 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3183: done (4 left) 2024-09-14T03:48:33.926 INFO:tasks.rados.rados.0.smithi110.stdout:3187: done (3 left) 2024-09-14T03:48:33.927 INFO:tasks.rados.rados.0.smithi110.stdout:3188: read oid 23 snap -1 2024-09-14T03:48:33.927 INFO:tasks.rados.rados.0.smithi110.stdout:3188: expect (ObjNum 967 snap 311 seq_num 967) 2024-09-14T03:48:33.927 INFO:tasks.rados.rados.0.smithi110.stdout:3189: read oid 31 snap 307 2024-09-14T03:48:33.927 INFO:tasks.rados.rados.0.smithi110.stdout:3189: expect deleted 2024-09-14T03:48:33.927 INFO:tasks.rados.rados.0.smithi110.stdout:3190: snap_remove snap 309 2024-09-14T03:48:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:33 smithi110 ceph-mon[88262]: pgmap v690: 121 pgs: 121 active+clean; 311 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 5.2 MiB/s wr, 14 op/s 2024-09-14T03:48:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:33 smithi110 ceph-mon[88262]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T03:48:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:33 smithi110 ceph-mon[88262]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T03:48:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:33 smithi110 ceph-mon[86423]: pgmap v690: 121 pgs: 121 active+clean; 311 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 5.2 MiB/s wr, 14 op/s 2024-09-14T03:48:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:33 smithi110 ceph-mon[86423]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T03:48:34.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:33 smithi110 ceph-mon[86423]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T03:48:34.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:33 smithi195 ceph-mon[82388]: pgmap v690: 121 pgs: 121 active+clean; 311 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 5.2 MiB/s wr, 14 op/s 2024-09-14T03:48:34.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:33 smithi195 ceph-mon[82388]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T03:48:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:33 smithi195 ceph-mon[82388]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T03:48:34.811 INFO:tasks.rados.rados.0.smithi110.stdout:3190: done (5 left) 2024-09-14T03:48:34.811 INFO:tasks.rados.rados.0.smithi110.stdout:3191: snap_remove snap 279 2024-09-14T03:48:35.815 INFO:tasks.rados.rados.0.smithi110.stdout:3191: done (5 left) 2024-09-14T03:48:35.816 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 315 2024-09-14T03:48:35.816 INFO:tasks.rados.rados.0.smithi110.stdout:3192: seq_num 983 ranges {1785856=466944,2252800=712704,2965504=434176,3399680=458752} 2024-09-14T03:48:35.829 INFO:tasks.rados.rados.0.smithi110.stdout:3192: writing smithi11075562-10 from 1785856 to 2252800 tid 1 2024-09-14T03:48:35.834 INFO:tasks.rados.rados.0.smithi110.stdout:3192: writing smithi11075562-10 from 2252800 to 2965504 tid 2 2024-09-14T03:48:35.837 INFO:tasks.rados.rados.0.smithi110.stdout:3192: writing smithi11075562-10 from 2965504 to 3399680 tid 3 2024-09-14T03:48:35.841 INFO:tasks.rados.rados.0.smithi110.stdout:3192: writing smithi11075562-10 from 3399680 to 3858432 tid 4 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3193: rollback oid 14 current snap is 315 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 14 to 307 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3184: finishing write tid 6 to smithi11075562-12 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2238 (ObjNum 980 snap 315 seq_num 980) dirty exists 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3184: left oid 12 (ObjNum 980 snap 315 seq_num 980) 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3185: finishing write tid 1 to smithi11075562-32 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3185: finishing write tid 2 to smithi11075562-32 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3185: finishing write tid 3 to smithi11075562-32 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3185: finishing write tid 4 to smithi11075562-32 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3184: done (6 left) 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3194: delete oid 25 current snap is 315 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3185: finishing write tid 5 to smithi11075562-32 2024-09-14T03:48:35.842 INFO:tasks.rados.rados.0.smithi110.stdout:3185: finishing write tid 6 to smithi11075562-32 2024-09-14T03:48:35.843 INFO:tasks.rados.rados.0.smithi110.stdout:3186: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:48:35.843 INFO:tasks.rados.rados.0.smithi110.stdout:3186: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:48:35.843 INFO:tasks.rados.rados.0.smithi110.stdout:3185: finishing write tid 7 to smithi11075562-32 2024-09-14T03:48:35.843 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2012 (ObjNum 981 snap 315 seq_num 981) dirty exists 2024-09-14T03:48:35.843 INFO:tasks.rados.rados.0.smithi110.stdout:3185: left oid 32 (ObjNum 981 snap 315 seq_num 981) 2024-09-14T03:48:35.843 INFO:tasks.rados.rados.0.smithi110.stdout:3186: finishing rollback tid 2 to smithi11075562-30 2024-09-14T03:48:35.843 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 2015 (ObjNum 962 snap 310 seq_num 962) dirty exists 2024-09-14T03:48:35.843 INFO:tasks.rados.rados.0.smithi110.stdout:3188: expect (ObjNum 967 snap 311 seq_num 967) 2024-09-14T03:48:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:35 smithi110 ceph-mon[86423]: pgmap v693: 121 pgs: 121 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.5 MiB/s wr, 4 op/s 2024-09-14T03:48:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:35 smithi110 ceph-mon[86423]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T03:48:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:35 smithi110 ceph-mon[88262]: pgmap v693: 121 pgs: 121 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.5 MiB/s wr, 4 op/s 2024-09-14T03:48:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:35 smithi110 ceph-mon[88262]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T03:48:36.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:35 smithi195 ceph-mon[82388]: pgmap v693: 121 pgs: 121 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.5 MiB/s wr, 4 op/s 2024-09-14T03:48:36.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:35 smithi195 ceph-mon[82388]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T03:48:36.310 INFO:tasks.rados.rados.0.smithi110.stdout:3192: finishing write tid 1 to smithi11075562-10 2024-09-14T03:48:36.310 INFO:tasks.rados.rados.0.smithi110.stdout:3192: finishing write tid 2 to smithi11075562-10 2024-09-14T03:48:36.310 INFO:tasks.rados.rados.0.smithi110.stdout:3192: finishing write tid 3 to smithi11075562-10 2024-09-14T03:48:36.310 INFO:tasks.rados.rados.0.smithi110.stdout:3192: finishing write tid 4 to smithi11075562-10 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3193: finishing rollback tid 0 to smithi11075562-14 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3193: finishing rollback tid 1 to smithi11075562-14 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1899 (ObjNum 49 snap 0 seq_num 1) dirty dne 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3192: finishing write tid 5 to smithi11075562-10 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3192: finishing write tid 6 to smithi11075562-10 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 1996 (ObjNum 983 snap 315 seq_num 983) dirty exists 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3192: left oid 10 (ObjNum 983 snap 315 seq_num 983) 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3185: done (6 left) 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3186: done (5 left) 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3188: done (4 left) 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3189: done (3 left) 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3192: done (2 left) 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3193: done (1 left) 2024-09-14T03:48:36.311 INFO:tasks.rados.rados.0.smithi110.stdout:3194: done (0 left) 2024-09-14T03:48:36.312 INFO:tasks.rados.rados.0.smithi110.stdout:3195: snap_remove snap 311 2024-09-14T03:48:36.825 INFO:tasks.rados.rados.0.smithi110.stdout:3195: done (0 left) 2024-09-14T03:48:36.825 INFO:tasks.rados.rados.0.smithi110.stdout:3196: copy_from oid 6 from oid 30 current snap is 315 2024-09-14T03:48:36.825 INFO:tasks.rados.rados.0.smithi110.stdout:3197: snap_remove snap 310 2024-09-14T03:48:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:36 smithi110 ceph-mon[88262]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T03:48:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:36 smithi110 ceph-mon[88262]: pgmap v696: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 311 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.7 MiB/s wr, 31 op/s 2024-09-14T03:48:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:36 smithi110 ceph-mon[86423]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T03:48:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:36 smithi110 ceph-mon[86423]: pgmap v696: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 311 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.7 MiB/s wr, 31 op/s 2024-09-14T03:48:37.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:36 smithi195 ceph-mon[82388]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T03:48:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:36 smithi195 ceph-mon[82388]: pgmap v696: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 311 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.7 MiB/s wr, 31 op/s 2024-09-14T03:48:37.834 INFO:tasks.rados.rados.0.smithi110.stdout:3197: done (1 left) 2024-09-14T03:48:37.834 INFO:tasks.rados.rados.0.smithi110.stdout:3198: read oid 1 snap -1 2024-09-14T03:48:37.834 INFO:tasks.rados.rados.0.smithi110.stdout:3198: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-14T03:48:37.834 INFO:tasks.rados.rados.0.smithi110.stdout:3199: read oid 21 snap -1 2024-09-14T03:48:37.835 INFO:tasks.rados.rados.0.smithi110.stdout:3199: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:37.835 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 315 2024-09-14T03:48:37.835 INFO:tasks.rados.rados.0.smithi110.stdout:3200: seq_num 985 ranges {6053888=499712,6553600=483328,7036928=450560,7487488=581632,8069120=90112} 2024-09-14T03:48:37.871 INFO:tasks.rados.rados.0.smithi110.stdout:3200: writing smithi11075562-4 from 6053888 to 6553600 tid 1 2024-09-14T03:48:37.874 INFO:tasks.rados.rados.0.smithi110.stdout:3200: writing smithi11075562-4 from 6553600 to 7036928 tid 2 2024-09-14T03:48:37.877 INFO:tasks.rados.rados.0.smithi110.stdout:3200: writing smithi11075562-4 from 7036928 to 7487488 tid 3 2024-09-14T03:48:37.880 INFO:tasks.rados.rados.0.smithi110.stdout:3200: writing smithi11075562-4 from 7487488 to 8069120 tid 4 2024-09-14T03:48:37.881 INFO:tasks.rados.rados.0.smithi110.stdout:3200: writing smithi11075562-4 from 8069120 to 8159232 tid 5 2024-09-14T03:48:37.881 INFO:tasks.rados.rados.0.smithi110.stdout:3201: snap_remove snap 307 2024-09-14T03:48:37.881 INFO:tasks.rados.rados.0.smithi110.stdout:3196: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:48:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:37 smithi110 ceph-mon[88262]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T03:48:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:37 smithi110 ceph-mon[86423]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T03:48:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:37 smithi195 ceph-mon[82388]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T03:48:38.794 INFO:tasks.rados.rados.0.smithi110.stdout:3201: done (4 left) 2024-09-14T03:48:38.794 INFO:tasks.rados.rados.0.smithi110.stdout:3202: delete oid 41 current snap is 315 2024-09-14T03:48:38.794 INFO:tasks.rados.rados.0.smithi110.stdout:3196: finishing copy_from to smithi11075562-6 2024-09-14T03:48:38.794 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2015 (ObjNum 962 snap 310 seq_num 962) dirty exists 2024-09-14T03:48:38.794 INFO:tasks.rados.rados.0.smithi110.stdout:3198: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-14T03:48:38.879 INFO:tasks.rados.rados.0.smithi110.stdout:3199: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:38 smithi110 ceph-mon[86423]: pgmap v698: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 289 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 31 op/s 2024-09-14T03:48:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:38 smithi110 ceph-mon[86423]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T03:48:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:38 smithi110 ceph-mon[86423]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T03:48:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:38 smithi110 ceph-mon[88262]: pgmap v698: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 289 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 31 op/s 2024-09-14T03:48:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:38 smithi110 ceph-mon[88262]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T03:48:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:38 smithi110 ceph-mon[88262]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T03:48:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:38 smithi195 ceph-mon[82388]: pgmap v698: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 118 active+clean; 289 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 31 op/s 2024-09-14T03:48:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:38 smithi195 ceph-mon[82388]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T03:48:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:38 smithi195 ceph-mon[82388]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T03:48:39.224 INFO:tasks.rados.rados.0.smithi110.stdout:3200: finishing write tid 1 to smithi11075562-4 2024-09-14T03:48:39.224 INFO:tasks.rados.rados.0.smithi110.stdout:3200: finishing write tid 2 to smithi11075562-4 2024-09-14T03:48:39.224 INFO:tasks.rados.rados.0.smithi110.stdout:3200: finishing write tid 3 to smithi11075562-4 2024-09-14T03:48:39.224 INFO:tasks.rados.rados.0.smithi110.stdout:3200: finishing write tid 4 to smithi11075562-4 2024-09-14T03:48:39.224 INFO:tasks.rados.rados.0.smithi110.stdout:3200: finishing write tid 5 to smithi11075562-4 2024-09-14T03:48:39.224 INFO:tasks.rados.rados.0.smithi110.stdout:3200: finishing write tid 6 to smithi11075562-4 2024-09-14T03:48:39.224 INFO:tasks.rados.rados.0.smithi110.stdout:3200: finishing write tid 7 to smithi11075562-4 2024-09-14T03:48:39.224 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2129 (ObjNum 985 snap 315 seq_num 985) dirty exists 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3200: left oid 4 (ObjNum 985 snap 315 seq_num 985) 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3196: done (4 left) 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3198: done (3 left) 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3199: done (2 left) 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3200: done (1 left) 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3202: done (0 left) 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3203: rollback oid 32 current snap is 315 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 314 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3204: read oid 10 snap -1 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3204: expect (ObjNum 983 snap 315 seq_num 983) 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3205: copy_from oid 48 from oid 24 current snap is 315 2024-09-14T03:48:39.225 INFO:tasks.rados.rados.0.smithi110.stdout:3206: snap_create 2024-09-14T03:48:39.226 INFO:tasks.rados.rados.0.smithi110.stdout:3203: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:48:39.226 INFO:tasks.rados.rados.0.smithi110.stdout:3203: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:48:39.226 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2017 (ObjNum 129 snap 0 seq_num 65516096) dirty dne 2024-09-14T03:48:39.238 INFO:tasks.rados.rados.0.smithi110.stdout:3204: expect (ObjNum 983 snap 315 seq_num 983) 2024-09-14T03:48:39.391 INFO:tasks.rados.rados.0.smithi110.stdout:3205: finishing copy_from racing read to smithi11075562-48 2024-09-14T03:48:39.391 INFO:tasks.rados.rados.0.smithi110.stdout:3205: finishing copy_from to smithi11075562-48 2024-09-14T03:48:39.391 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1991 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:48:39.837 INFO:tasks.rados.rados.0.smithi110.stdout:3203: done (3 left) 2024-09-14T03:48:39.837 INFO:tasks.rados.rados.0.smithi110.stdout:3204: done (2 left) 2024-09-14T03:48:39.837 INFO:tasks.rados.rados.0.smithi110.stdout:3205: done (1 left) 2024-09-14T03:48:39.837 INFO:tasks.rados.rados.0.smithi110.stdout:3206: done (0 left) 2024-09-14T03:48:39.837 INFO:tasks.rados.rados.0.smithi110.stdout:3207: snap_create 2024-09-14T03:48:40.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:48:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:48:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:48:40.839 INFO:tasks.rados.rados.0.smithi110.stdout:3207: done (0 left) 2024-09-14T03:48:40.840 INFO:tasks.rados.rados.0.smithi110.stdout:3208: snap_create 2024-09-14T03:48:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:40 smithi110 ceph-mon[86423]: pgmap v701: 121 pgs: 121 active+clean; 228 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.7 MiB/s wr, 7 op/s 2024-09-14T03:48:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:40 smithi110 ceph-mon[86423]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T03:48:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:40 smithi110 ceph-mon[88262]: pgmap v701: 121 pgs: 121 active+clean; 228 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.7 MiB/s wr, 7 op/s 2024-09-14T03:48:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:40 smithi110 ceph-mon[88262]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T03:48:41.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:40 smithi195 ceph-mon[82388]: pgmap v701: 121 pgs: 121 active+clean; 228 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.7 MiB/s wr, 7 op/s 2024-09-14T03:48:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:40 smithi195 ceph-mon[82388]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T03:48:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:3208: done (0 left) 2024-09-14T03:48:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:3209: rmattr oid 31 current snap is 318 2024-09-14T03:48:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:3209: done (0 left) 2024-09-14T03:48:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 318 2024-09-14T03:48:41.842 INFO:tasks.rados.rados.0.smithi110.stdout:3210: seq_num 987 ranges {7544832=532480,8077312=647168,8724480=417792,9142272=409600,9551872=131072} 2024-09-14T03:48:41.846 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:48:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:48:41] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-14T03:48:41.885 INFO:tasks.rados.rados.0.smithi110.stdout:3210: writing smithi11075562-24 from 7544832 to 8077312 tid 1 2024-09-14T03:48:41.889 INFO:tasks.rados.rados.0.smithi110.stdout:3210: writing smithi11075562-24 from 8077312 to 8724480 tid 2 2024-09-14T03:48:41.891 INFO:tasks.rados.rados.0.smithi110.stdout:3210: writing smithi11075562-24 from 8724480 to 9142272 tid 3 2024-09-14T03:48:41.894 INFO:tasks.rados.rados.0.smithi110.stdout:3210: writing smithi11075562-24 from 9142272 to 9551872 tid 4 2024-09-14T03:48:41.895 INFO:tasks.rados.rados.0.smithi110.stdout:3210: writing smithi11075562-24 from 9551872 to 9682944 tid 5 2024-09-14T03:48:41.895 INFO:tasks.rados.rados.0.smithi110.stdout:3211: copy_from oid 13 from oid 50 current snap is 318 2024-09-14T03:48:41.895 INFO:tasks.rados.rados.0.smithi110.stdout:3210: finishing write tid 1 to smithi11075562-24 2024-09-14T03:48:41.895 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 318 2024-09-14T03:48:41.895 INFO:tasks.rados.rados.0.smithi110.stdout:3212: seq_num 988 ranges {9265152=557056,9822208=532480,10354688=622592,10977280=450560} 2024-09-14T03:48:41.947 INFO:tasks.rados.rados.0.smithi110.stdout:3212: writing smithi11075562-6 from 9265152 to 9822208 tid 1 2024-09-14T03:48:41.950 INFO:tasks.rados.rados.0.smithi110.stdout:3212: writing smithi11075562-6 from 9822208 to 10354688 tid 2 2024-09-14T03:48:41.954 INFO:tasks.rados.rados.0.smithi110.stdout:3212: writing smithi11075562-6 from 10354688 to 10977280 tid 3 2024-09-14T03:48:41.957 INFO:tasks.rados.rados.0.smithi110.stdout:3212: writing smithi11075562-6 from 10977280 to 11427840 tid 4 2024-09-14T03:48:41.957 INFO:tasks.rados.rados.0.smithi110.stdout:3213: copy_from oid 2 from oid 10 current snap is 318 2024-09-14T03:48:41.957 INFO:tasks.rados.rados.0.smithi110.stdout:3210: finishing write tid 2 to smithi11075562-24 2024-09-14T03:48:41.957 INFO:tasks.rados.rados.0.smithi110.stdout:3210: finishing write tid 3 to smithi11075562-24 2024-09-14T03:48:41.957 INFO:tasks.rados.rados.0.smithi110.stdout:3210: finishing write tid 4 to smithi11075562-24 2024-09-14T03:48:41.957 INFO:tasks.rados.rados.0.smithi110.stdout:3210: finishing write tid 5 to smithi11075562-24 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3214: read oid 18 snap -1 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3214: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3215: delete oid 12 current snap is 318 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3210: finishing write tid 6 to smithi11075562-24 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3210: finishing write tid 7 to smithi11075562-24 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2002 (ObjNum 987 snap 318 seq_num 987) dirty exists 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3210: left oid 24 (ObjNum 987 snap 318 seq_num 987) 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3211: finishing copy_from racing read to smithi11075562-13 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3211: finishing copy_from to smithi11075562-13 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 1510 (ObjNum 972 snap 314 seq_num 972) dirty exists 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3212: finishing write tid 1 to smithi11075562-6 2024-09-14T03:48:41.958 INFO:tasks.rados.rados.0.smithi110.stdout:3212: finishing write tid 2 to smithi11075562-6 2024-09-14T03:48:41.960 INFO:tasks.rados.rados.0.smithi110.stdout:3210: done (5 left) 2024-09-14T03:48:41.960 INFO:tasks.rados.rados.0.smithi110.stdout:3211: done (4 left) 2024-09-14T03:48:41.960 INFO:tasks.rados.rados.0.smithi110.stdout:3215: done (3 left) 2024-09-14T03:48:41.960 INFO:tasks.rados.rados.0.smithi110.stdout:3216: read oid 25 snap 317 2024-09-14T03:48:41.961 INFO:tasks.rados.rados.0.smithi110.stdout:3216: expect deleted 2024-09-14T03:48:41.961 INFO:tasks.rados.rados.0.smithi110.stdout:3217: copy_from oid 46 from oid 31 current snap is 318 2024-09-14T03:48:41.961 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 318 2024-09-14T03:48:41.961 INFO:tasks.rados.rados.0.smithi110.stdout:3218: seq_num 990 ranges {0=589824,589824=753664,1343488=638976,1982464=212992} 2024-09-14T03:48:41.964 INFO:tasks.rados.rados.0.smithi110.stdout:3218: writing smithi11075562-32 from 0 to 589824 tid 1 2024-09-14T03:48:41.969 INFO:tasks.rados.rados.0.smithi110.stdout:3218: writing smithi11075562-32 from 589824 to 1343488 tid 2 2024-09-14T03:48:41.973 INFO:tasks.rados.rados.0.smithi110.stdout:3218: writing smithi11075562-32 from 1343488 to 1982464 tid 3 2024-09-14T03:48:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3218: writing smithi11075562-32 from 1982464 to 2195456 tid 4 2024-09-14T03:48:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 318 2024-09-14T03:48:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3212: finishing write tid 3 to smithi11075562-6 2024-09-14T03:48:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3212: finishing write tid 4 to smithi11075562-6 2024-09-14T03:48:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3212: finishing write tid 5 to smithi11075562-6 2024-09-14T03:48:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3214: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:41.979 INFO:tasks.rados.rados.0.smithi110.stdout:3212: finishing write tid 6 to smithi11075562-6 2024-09-14T03:48:41.979 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2127 (ObjNum 988 snap 318 seq_num 988) dirty exists 2024-09-14T03:48:41.979 INFO:tasks.rados.rados.0.smithi110.stdout:3212: left oid 6 (ObjNum 988 snap 318 seq_num 988) 2024-09-14T03:48:41.980 INFO:tasks.rados.rados.0.smithi110.stdout:3218: finishing write tid 1 to smithi11075562-32 2024-09-14T03:48:41.980 INFO:tasks.rados.rados.0.smithi110.stdout:3217: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:48:41.980 INFO:tasks.rados.rados.0.smithi110.stdout:3218: finishing write tid 2 to smithi11075562-32 2024-09-14T03:48:41.980 INFO:tasks.rados.rados.0.smithi110.stdout:3217: finishing copy_from to smithi11075562-46 2024-09-14T03:48:41.980 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 2123 (ObjNum 964 snap 311 seq_num 964) dirty exists 2024-09-14T03:48:41.980 INFO:tasks.rados.rados.0.smithi110.stdout:3213: finishing copy_from racing read to smithi11075562-2 2024-09-14T03:48:41.980 INFO:tasks.rados.rados.0.smithi110.stdout:3219: seq_num 991 ranges {0=606208,606208=638976,1245184=794624,2039808=172032} 2024-09-14T03:48:41.983 INFO:tasks.rados.rados.0.smithi110.stdout:3219: writing smithi11075562-47 from 0 to 606208 tid 1 2024-09-14T03:48:41.986 INFO:tasks.rados.rados.0.smithi110.stdout:3219: writing smithi11075562-47 from 606208 to 1245184 tid 2 2024-09-14T03:48:41.990 INFO:tasks.rados.rados.0.smithi110.stdout:3219: writing smithi11075562-47 from 1245184 to 2039808 tid 3 2024-09-14T03:48:41.992 INFO:tasks.rados.rados.0.smithi110.stdout:3219: writing smithi11075562-47 from 2039808 to 2211840 tid 4 2024-09-14T03:48:41.992 INFO:tasks.rados.rados.0.smithi110.stdout:3212: done (6 left) 2024-09-14T03:48:41.992 INFO:tasks.rados.rados.0.smithi110.stdout:3214: done (5 left) 2024-09-14T03:48:41.992 INFO:tasks.rados.rados.0.smithi110.stdout:3216: done (4 left) 2024-09-14T03:48:41.992 INFO:tasks.rados.rados.0.smithi110.stdout:3217: done (3 left) 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3220: copy_from oid 27 from oid 34 current snap is 318 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3213: finishing copy_from to smithi11075562-2 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 1996 (ObjNum 983 snap 315 seq_num 983) dirty exists 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3218: finishing write tid 3 to smithi11075562-32 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3218: finishing write tid 4 to smithi11075562-32 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3218: finishing write tid 5 to smithi11075562-32 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3218: finishing write tid 6 to smithi11075562-32 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2022 (ObjNum 990 snap 318 seq_num 990) dirty exists 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3218: left oid 32 (ObjNum 990 snap 318 seq_num 990) 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3219: finishing write tid 1 to smithi11075562-47 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3219: finishing write tid 2 to smithi11075562-47 2024-09-14T03:48:41.993 INFO:tasks.rados.rados.0.smithi110.stdout:3213: done (3 left) 2024-09-14T03:48:41.994 INFO:tasks.rados.rados.0.smithi110.stdout:3218: done (2 left) 2024-09-14T03:48:41.994 INFO:tasks.rados.rados.0.smithi110.stdout:3221: snap_create 2024-09-14T03:48:41.996 INFO:tasks.rados.rados.0.smithi110.stdout:3220: finishing copy_from racing read to smithi11075562-27 2024-09-14T03:48:41.996 INFO:tasks.rados.rados.0.smithi110.stdout:3219: finishing write tid 3 to smithi11075562-47 2024-09-14T03:48:41.996 INFO:tasks.rados.rados.0.smithi110.stdout:3219: finishing write tid 4 to smithi11075562-47 2024-09-14T03:48:41.997 INFO:tasks.rados.rados.0.smithi110.stdout:3220: finishing copy_from to smithi11075562-27 2024-09-14T03:48:41.997 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1615 (ObjNum 910 snap 297 seq_num 910) dirty exists 2024-09-14T03:48:41.997 INFO:tasks.rados.rados.0.smithi110.stdout:3219: finishing write tid 5 to smithi11075562-47 2024-09-14T03:48:41.998 INFO:tasks.rados.rados.0.smithi110.stdout:3219: finishing write tid 6 to smithi11075562-47 2024-09-14T03:48:41.999 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 2294 (ObjNum 991 snap 318 seq_num 991) dirty exists 2024-09-14T03:48:41.999 INFO:tasks.rados.rados.0.smithi110.stdout:3219: left oid 47 (ObjNum 991 snap 318 seq_num 991) 2024-09-14T03:48:42.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:41 smithi110 ceph-mon[88262]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T03:48:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:41 smithi110 ceph-mon[86423]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T03:48:42.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:41 smithi195 ceph-mon[82388]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T03:48:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3219: done (2 left) 2024-09-14T03:48:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3220: done (1 left) 2024-09-14T03:48:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3221: done (0 left) 2024-09-14T03:48:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3222: read oid 38 snap 317 2024-09-14T03:48:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3222: expect (ObjNum 963 snap 310 seq_num 963) 2024-09-14T03:48:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3223: delete oid 20 current snap is 319 2024-09-14T03:48:42.860 INFO:tasks.rados.rados.0.smithi110.stdout:3223: done (1 left) 2024-09-14T03:48:42.860 INFO:tasks.rados.rados.0.smithi110.stdout:3224: snap_create 2024-09-14T03:48:42.864 INFO:tasks.rados.rados.0.smithi110.stdout:3222: expect (ObjNum 963 snap 310 seq_num 963) 2024-09-14T03:48:43.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:42 smithi195 ceph-mon[82388]: pgmap v704: 121 pgs: 121 active+clean; 177 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 4.5 MiB/s wr, 7 op/s 2024-09-14T03:48:43.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:42 smithi195 ceph-mon[82388]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T03:48:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:42 smithi110 ceph-mon[88262]: pgmap v704: 121 pgs: 121 active+clean; 177 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 4.5 MiB/s wr, 7 op/s 2024-09-14T03:48:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:42 smithi110 ceph-mon[88262]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T03:48:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:42 smithi110 ceph-mon[86423]: pgmap v704: 121 pgs: 121 active+clean; 177 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 4.5 MiB/s wr, 7 op/s 2024-09-14T03:48:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:42 smithi110 ceph-mon[86423]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T03:48:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:43.794 INFO:tasks.rados.rados.0.smithi110.stdout:3222: done (1 left) 2024-09-14T03:48:43.794 INFO:tasks.rados.rados.0.smithi110.stdout:3224: done (0 left) 2024-09-14T03:48:43.794 INFO:tasks.rados.rados.0.smithi110.stdout:3225: rollback oid 31 current snap is 320 2024-09-14T03:48:43.794 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 319 2024-09-14T03:48:43.795 INFO:tasks.rados.rados.0.smithi110.stdout:3226: read oid 45 snap -1 2024-09-14T03:48:43.795 INFO:tasks.rados.rados.0.smithi110.stdout:3226: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:48:43.795 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 34 current snap is 320 2024-09-14T03:48:43.795 INFO:tasks.rados.rados.0.smithi110.stdout:3227: seq_num 993 ranges {843776=638976,1482752=409600,1892352=409600,2301952=688128,2990080=98304} 2024-09-14T03:48:43.803 INFO:tasks.rados.rados.0.smithi110.stdout:3227: writing smithi11075562-34 from 843776 to 1482752 tid 1 2024-09-14T03:48:43.805 INFO:tasks.rados.rados.0.smithi110.stdout:3227: writing smithi11075562-34 from 1482752 to 1892352 tid 2 2024-09-14T03:48:43.807 INFO:tasks.rados.rados.0.smithi110.stdout:3227: writing smithi11075562-34 from 1892352 to 2301952 tid 3 2024-09-14T03:48:43.811 INFO:tasks.rados.rados.0.smithi110.stdout:3227: writing smithi11075562-34 from 2301952 to 2990080 tid 4 2024-09-14T03:48:43.812 INFO:tasks.rados.rados.0.smithi110.stdout:3227: writing smithi11075562-34 from 2990080 to 3088384 tid 5 2024-09-14T03:48:43.812 INFO:tasks.rados.rados.0.smithi110.stdout:3228: snap_create 2024-09-14T03:48:43.812 INFO:tasks.rados.rados.0.smithi110.stdout:3225: finishing rollback tid 0 to smithi11075562-31 2024-09-14T03:48:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:3225: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:48:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:3225: finishing rollback tid 2 to smithi11075562-31 2024-09-14T03:48:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 2132 (ObjNum 964 snap 311 seq_num 964) dirty exists 2024-09-14T03:48:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:3226: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3227: finishing write tid 1 to smithi11075562-34 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3227: finishing write tid 2 to smithi11075562-34 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3227: finishing write tid 3 to smithi11075562-34 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3227: finishing write tid 4 to smithi11075562-34 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3227: finishing write tid 5 to smithi11075562-34 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3227: finishing write tid 6 to smithi11075562-34 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3227: finishing write tid 7 to smithi11075562-34 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 2138 (ObjNum 993 snap 320 seq_num 993) dirty exists 2024-09-14T03:48:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3227: left oid 34 (ObjNum 993 snap 320 seq_num 993) 2024-09-14T03:48:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:43 smithi110 ceph-mon[88262]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T03:48:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:43 smithi110 ceph-mon[88262]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T03:48:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:43 smithi110 ceph-mon[86423]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T03:48:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:43 smithi110 ceph-mon[86423]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T03:48:44.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:43 smithi195 ceph-mon[82388]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T03:48:44.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:43 smithi195 ceph-mon[82388]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T03:48:44.805 INFO:tasks.rados.rados.0.smithi110.stdout:3225: done (3 left) 2024-09-14T03:48:44.805 INFO:tasks.rados.rados.0.smithi110.stdout:3226: done (2 left) 2024-09-14T03:48:44.805 INFO:tasks.rados.rados.0.smithi110.stdout:3227: done (1 left) 2024-09-14T03:48:44.805 INFO:tasks.rados.rados.0.smithi110.stdout:3228: done (0 left) 2024-09-14T03:48:44.805 INFO:tasks.rados.rados.0.smithi110.stdout:3229: rollback oid 46 current snap is 321 2024-09-14T03:48:44.805 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 46 to 316 2024-09-14T03:48:44.805 INFO:tasks.rados.rados.0.smithi110.stdout:3230: read oid 21 snap -1 2024-09-14T03:48:44.806 INFO:tasks.rados.rados.0.smithi110.stdout:3230: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:44.806 INFO:tasks.rados.rados.0.smithi110.stdout:3231: rollback oid 13 current snap is 321 2024-09-14T03:48:44.806 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 13 to 315 2024-09-14T03:48:44.806 INFO:tasks.rados.rados.0.smithi110.stdout:3232: read oid 11 snap -1 2024-09-14T03:48:44.806 INFO:tasks.rados.rados.0.smithi110.stdout:3232: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:44.806 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 321 2024-09-14T03:48:44.806 INFO:tasks.rados.rados.0.smithi110.stdout:3233: seq_num 994 ranges {0=655360,655360=688128,1343488=622592,1966080=294912} 2024-09-14T03:48:44.808 INFO:tasks.rados.rados.0.smithi110.stdout:3233: writing smithi11075562-9 from 0 to 655360 tid 1 2024-09-14T03:48:44.813 INFO:tasks.rados.rados.0.smithi110.stdout:3233: writing smithi11075562-9 from 655360 to 1343488 tid 2 2024-09-14T03:48:44.817 INFO:tasks.rados.rados.0.smithi110.stdout:3233: writing smithi11075562-9 from 1343488 to 1966080 tid 3 2024-09-14T03:48:44.820 INFO:tasks.rados.rados.0.smithi110.stdout:3233: writing smithi11075562-9 from 1966080 to 2260992 tid 4 2024-09-14T03:48:44.820 INFO:tasks.rados.rados.0.smithi110.stdout:3234: copy_from oid 30 from oid 1 current snap is 321 2024-09-14T03:48:44.820 INFO:tasks.rados.rados.0.smithi110.stdout:3231: finishing rollback tid 0 to smithi11075562-13 2024-09-14T03:48:44.820 INFO:tasks.rados.rados.0.smithi110.stdout:3231: finishing rollback tid 1 to smithi11075562-13 2024-09-14T03:48:44.820 INFO:tasks.rados.rados.0.smithi110.stdout:3231: finishing rollback tid 2 to smithi11075562-13 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 2021 (ObjNum 930 snap 302 seq_num 930) dirty exists 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:3229: finishing rollback tid 0 to smithi11075562-46 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:3229: finishing rollback tid 1 to smithi11075562-46 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:3229: finishing rollback tid 2 to smithi11075562-46 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 2127 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:3229: done (5 left) 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:3231: done (4 left) 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:3235: read oid 44 snap -1 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:3235: expect (ObjNum 976 snap 314 seq_num 976) 2024-09-14T03:48:44.821 INFO:tasks.rados.rados.0.smithi110.stdout:3236: snap_create 2024-09-14T03:48:44.822 INFO:tasks.rados.rados.0.smithi110.stdout:3233: finishing write tid 1 to smithi11075562-9 2024-09-14T03:48:44.823 INFO:tasks.rados.rados.0.smithi110.stdout:3232: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:48:44.901 INFO:tasks.rados.rados.0.smithi110.stdout:3233: finishing write tid 2 to smithi11075562-9 2024-09-14T03:48:44.901 INFO:tasks.rados.rados.0.smithi110.stdout:3233: finishing write tid 3 to smithi11075562-9 2024-09-14T03:48:44.901 INFO:tasks.rados.rados.0.smithi110.stdout:3233: finishing write tid 4 to smithi11075562-9 2024-09-14T03:48:44.901 INFO:tasks.rados.rados.0.smithi110.stdout:3233: finishing write tid 5 to smithi11075562-9 2024-09-14T03:48:44.901 INFO:tasks.rados.rados.0.smithi110.stdout:3233: finishing write tid 6 to smithi11075562-9 2024-09-14T03:48:44.901 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 2132 (ObjNum 994 snap 321 seq_num 994) dirty exists 2024-09-14T03:48:44.901 INFO:tasks.rados.rados.0.smithi110.stdout:3233: left oid 9 (ObjNum 994 snap 321 seq_num 994) 2024-09-14T03:48:44.902 INFO:tasks.rados.rados.0.smithi110.stdout:3230: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:45.241 INFO:tasks.rados.rados.0.smithi110.stdout:3234: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:48:45.241 INFO:tasks.rados.rados.0.smithi110.stdout:3234: finishing copy_from to smithi11075562-30 2024-09-14T03:48:45.241 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 1997 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:48:45.241 INFO:tasks.rados.rados.0.smithi110.stdout:3235: expect (ObjNum 976 snap 314 seq_num 976) 2024-09-14T03:48:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:45 smithi110 ceph-mon[88262]: pgmap v708: 121 pgs: 121 active+clean; 179 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-14T03:48:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:45 smithi110 ceph-mon[88262]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T03:48:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:45 smithi110 ceph-mon[86423]: pgmap v708: 121 pgs: 121 active+clean; 179 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-14T03:48:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:45 smithi110 ceph-mon[86423]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T03:48:46.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:45 smithi195 ceph-mon[82388]: pgmap v708: 121 pgs: 121 active+clean; 179 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-14T03:48:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:45 smithi195 ceph-mon[82388]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T03:48:46.333 INFO:tasks.rados.rados.0.smithi110.stdout:3230: done (5 left) 2024-09-14T03:48:46.334 INFO:tasks.rados.rados.0.smithi110.stdout:3232: done (4 left) 2024-09-14T03:48:46.334 INFO:tasks.rados.rados.0.smithi110.stdout:3233: done (3 left) 2024-09-14T03:48:46.334 INFO:tasks.rados.rados.0.smithi110.stdout:3234: done (2 left) 2024-09-14T03:48:46.334 INFO:tasks.rados.rados.0.smithi110.stdout:3235: done (1 left) 2024-09-14T03:48:46.334 INFO:tasks.rados.rados.0.smithi110.stdout:3236: done (0 left) 2024-09-14T03:48:46.334 INFO:tasks.rados.rados.0.smithi110.stdout:3237: delete oid 13 current snap is 322 2024-09-14T03:48:46.340 INFO:tasks.rados.rados.0.smithi110.stdout:3237: done (0 left) 2024-09-14T03:48:46.340 INFO:tasks.rados.rados.0.smithi110.stdout:3238: copy_from oid 2 from oid 26 current snap is 322 2024-09-14T03:48:46.340 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 322 2024-09-14T03:48:46.340 INFO:tasks.rados.rados.0.smithi110.stdout:3239: seq_num 996 ranges {2195456=688128,2883584=507904,3391488=581632,3973120=516096} 2024-09-14T03:48:46.356 INFO:tasks.rados.rados.0.smithi110.stdout:3239: writing smithi11075562-32 from 2195456 to 2883584 tid 1 2024-09-14T03:48:46.358 INFO:tasks.rados.rados.0.smithi110.stdout:3239: writing smithi11075562-32 from 2883584 to 3391488 tid 2 2024-09-14T03:48:46.362 INFO:tasks.rados.rados.0.smithi110.stdout:3239: writing smithi11075562-32 from 3391488 to 3973120 tid 3 2024-09-14T03:48:46.366 INFO:tasks.rados.rados.0.smithi110.stdout:3239: writing smithi11075562-32 from 3973120 to 4489216 tid 4 2024-09-14T03:48:46.366 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 322 2024-09-14T03:48:46.366 INFO:tasks.rados.rados.0.smithi110.stdout:3239: finishing write tid 1 to smithi11075562-32 2024-09-14T03:48:46.366 INFO:tasks.rados.rados.0.smithi110.stdout:3239: finishing write tid 2 to smithi11075562-32 2024-09-14T03:48:46.366 INFO:tasks.rados.rados.0.smithi110.stdout:3239: finishing write tid 3 to smithi11075562-32 2024-09-14T03:48:46.366 INFO:tasks.rados.rados.0.smithi110.stdout:3240: seq_num 997 ranges {0=704512,704512=794624,1499136=794624,2293760=16384} 2024-09-14T03:48:46.370 INFO:tasks.rados.rados.0.smithi110.stdout:3240: writing smithi11075562-13 from 0 to 704512 tid 1 2024-09-14T03:48:46.375 INFO:tasks.rados.rados.0.smithi110.stdout:3240: writing smithi11075562-13 from 704512 to 1499136 tid 2 2024-09-14T03:48:46.381 INFO:tasks.rados.rados.0.smithi110.stdout:3240: writing smithi11075562-13 from 1499136 to 2293760 tid 3 2024-09-14T03:48:46.381 INFO:tasks.rados.rados.0.smithi110.stdout:3240: writing smithi11075562-13 from 2293760 to 2310144 tid 4 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3241: rollback oid 4 current snap is 322 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 4 to 319 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3239: finishing write tid 4 to smithi11075562-32 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3239: finishing write tid 5 to smithi11075562-32 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3239: finishing write tid 6 to smithi11075562-32 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2027 (ObjNum 996 snap 322 seq_num 996) dirty exists 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3239: left oid 32 (ObjNum 996 snap 322 seq_num 996) 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3239: done (3 left) 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3242: rmattr oid 20 current snap is 322 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3242: done (3 left) 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3243: setattr oid 41 current snap is 322 2024-09-14T03:48:46.382 INFO:tasks.rados.rados.0.smithi110.stdout:3240: finishing write tid 1 to smithi11075562-13 2024-09-14T03:48:46.383 INFO:tasks.rados.rados.0.smithi110.stdout:3240: finishing write tid 2 to smithi11075562-13 2024-09-14T03:48:46.383 INFO:tasks.rados.rados.0.smithi110.stdout:3244: snap_create 2024-09-14T03:48:46.387 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1901 (ObjNum 65 snap 0 seq_num 1416655974) dirty exists 2024-09-14T03:48:46.387 INFO:tasks.rados.rados.0.smithi110.stdout:3241: finishing rollback tid 0 to smithi11075562-4 2024-09-14T03:48:46.387 INFO:tasks.rados.rados.0.smithi110.stdout:3240: finishing write tid 3 to smithi11075562-13 2024-09-14T03:48:46.387 INFO:tasks.rados.rados.0.smithi110.stdout:3241: finishing rollback tid 1 to smithi11075562-4 2024-09-14T03:48:46.387 INFO:tasks.rados.rados.0.smithi110.stdout:3241: finishing rollback tid 2 to smithi11075562-4 2024-09-14T03:48:46.388 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2141 (ObjNum 985 snap 315 seq_num 985) dirty exists 2024-09-14T03:48:46.389 INFO:tasks.rados.rados.0.smithi110.stdout:3240: finishing write tid 4 to smithi11075562-13 2024-09-14T03:48:46.389 INFO:tasks.rados.rados.0.smithi110.stdout:3240: finishing write tid 5 to smithi11075562-13 2024-09-14T03:48:46.390 INFO:tasks.rados.rados.0.smithi110.stdout:3240: finishing write tid 6 to smithi11075562-13 2024-09-14T03:48:46.390 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 2027 (ObjNum 997 snap 322 seq_num 997) dirty exists 2024-09-14T03:48:46.390 INFO:tasks.rados.rados.0.smithi110.stdout:3240: left oid 13 (ObjNum 997 snap 322 seq_num 997) 2024-09-14T03:48:46.396 INFO:tasks.rados.rados.0.smithi110.stdout:3238: finishing copy_from racing read to smithi11075562-2 2024-09-14T03:48:46.398 INFO:tasks.rados.rados.0.smithi110.stdout:3238: finishing copy_from to smithi11075562-2 2024-09-14T03:48:46.398 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 1842 (ObjNum 974 snap 314 seq_num 974) dirty exists 2024-09-14T03:48:46.813 INFO:tasks.rados.rados.0.smithi110.stdout:3238: done (4 left) 2024-09-14T03:48:46.813 INFO:tasks.rados.rados.0.smithi110.stdout:3240: done (3 left) 2024-09-14T03:48:46.813 INFO:tasks.rados.rados.0.smithi110.stdout:3241: done (2 left) 2024-09-14T03:48:46.813 INFO:tasks.rados.rados.0.smithi110.stdout:3243: done (1 left) 2024-09-14T03:48:46.814 INFO:tasks.rados.rados.0.smithi110.stdout:3244: done (0 left) 2024-09-14T03:48:46.814 INFO:tasks.rados.rados.0.smithi110.stdout:3245: delete oid 42 current snap is 323 2024-09-14T03:48:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:3245: done (0 left) 2024-09-14T03:48:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 323 2024-09-14T03:48:46.818 INFO:tasks.rados.rados.0.smithi110.stdout:3246: seq_num 999 ranges {13598720=737280,14336000=565248,14901248=761856,15663104=278528} 2024-09-14T03:48:46.893 INFO:tasks.rados.rados.0.smithi110.stdout:3246: writing smithi11075562-26 from 13598720 to 14336000 tid 1 2024-09-14T03:48:46.896 INFO:tasks.rados.rados.0.smithi110.stdout:3246: writing smithi11075562-26 from 14336000 to 14901248 tid 2 2024-09-14T03:48:46.900 INFO:tasks.rados.rados.0.smithi110.stdout:3246: writing smithi11075562-26 from 14901248 to 15663104 tid 3 2024-09-14T03:48:46.902 INFO:tasks.rados.rados.0.smithi110.stdout:3246: writing smithi11075562-26 from 15663104 to 15941632 tid 4 2024-09-14T03:48:46.902 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 323 2024-09-14T03:48:46.902 INFO:tasks.rados.rados.0.smithi110.stdout:3247: seq_num 1000 ranges {0=753664,753664=442368,1196032=565248,1761280=458752,2220032=139264} 2024-09-14T03:48:46.906 INFO:tasks.rados.rados.0.smithi110.stdout:3247: writing smithi11075562-42 from 0 to 753664 tid 1 2024-09-14T03:48:46.909 INFO:tasks.rados.rados.0.smithi110.stdout:3247: writing smithi11075562-42 from 753664 to 1196032 tid 2 2024-09-14T03:48:46.913 INFO:tasks.rados.rados.0.smithi110.stdout:3247: writing smithi11075562-42 from 1196032 to 1761280 tid 3 2024-09-14T03:48:46.915 INFO:tasks.rados.rados.0.smithi110.stdout:3247: writing smithi11075562-42 from 1761280 to 2220032 tid 4 2024-09-14T03:48:46.917 INFO:tasks.rados.rados.0.smithi110.stdout:3247: writing smithi11075562-42 from 2220032 to 2359296 tid 5 2024-09-14T03:48:46.917 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 323 2024-09-14T03:48:46.917 INFO:tasks.rados.rados.0.smithi110.stdout:3246: finishing write tid 1 to smithi11075562-26 2024-09-14T03:48:46.917 INFO:tasks.rados.rados.0.smithi110.stdout:3246: finishing write tid 2 to smithi11075562-26 2024-09-14T03:48:46.917 INFO:tasks.rados.rados.0.smithi110.stdout:3246: finishing write tid 3 to smithi11075562-26 2024-09-14T03:48:46.917 INFO:tasks.rados.rados.0.smithi110.stdout:3246: finishing write tid 4 to smithi11075562-26 2024-09-14T03:48:46.917 INFO:tasks.rados.rados.0.smithi110.stdout:3248: seq_num 1001 ranges {0=770048,770048=729088,1499136=770048,2269184=106496} 2024-09-14T03:48:46.921 INFO:tasks.rados.rados.0.smithi110.stdout:3248: writing smithi11075562-22 from 0 to 770048 tid 1 2024-09-14T03:48:46.926 INFO:tasks.rados.rados.0.smithi110.stdout:3248: writing smithi11075562-22 from 770048 to 1499136 tid 2 2024-09-14T03:48:46.930 INFO:tasks.rados.rados.0.smithi110.stdout:3248: writing smithi11075562-22 from 1499136 to 2269184 tid 3 2024-09-14T03:48:46.931 INFO:tasks.rados.rados.0.smithi110.stdout:3248: writing smithi11075562-22 from 2269184 to 2375680 tid 4 2024-09-14T03:48:46.931 INFO:tasks.rados.rados.0.smithi110.stdout:3249: delete oid 44 current snap is 323 2024-09-14T03:48:46.931 INFO:tasks.rados.rados.0.smithi110.stdout:3246: finishing write tid 5 to smithi11075562-26 2024-09-14T03:48:46.932 INFO:tasks.rados.rados.0.smithi110.stdout:3246: finishing write tid 6 to smithi11075562-26 2024-09-14T03:48:46.932 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1847 (ObjNum 999 snap 323 seq_num 999) dirty exists 2024-09-14T03:48:46.932 INFO:tasks.rados.rados.0.smithi110.stdout:3246: left oid 26 (ObjNum 999 snap 323 seq_num 999) 2024-09-14T03:48:46.932 INFO:tasks.rados.rados.0.smithi110.stdout:3247: finishing write tid 1 to smithi11075562-42 2024-09-14T03:48:46.932 INFO:tasks.rados.rados.0.smithi110.stdout:3247: finishing write tid 2 to smithi11075562-42 2024-09-14T03:48:46.932 INFO:tasks.rados.rados.0.smithi110.stdout:3247: finishing write tid 3 to smithi11075562-42 2024-09-14T03:48:46.932 INFO:tasks.rados.rados.0.smithi110.stdout:3247: finishing write tid 4 to smithi11075562-42 2024-09-14T03:48:46.932 INFO:tasks.rados.rados.0.smithi110.stdout:3247: finishing write tid 5 to smithi11075562-42 2024-09-14T03:48:46.933 INFO:tasks.rados.rados.0.smithi110.stdout:3247: finishing write tid 6 to smithi11075562-42 2024-09-14T03:48:46.933 INFO:tasks.rados.rados.0.smithi110.stdout:3247: finishing write tid 7 to smithi11075562-42 2024-09-14T03:48:46.933 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2034 (ObjNum 1000 snap 323 seq_num 1000) dirty exists 2024-09-14T03:48:46.933 INFO:tasks.rados.rados.0.smithi110.stdout:3247: left oid 42 (ObjNum 1000 snap 323 seq_num 1000) 2024-09-14T03:48:46.933 INFO:tasks.rados.rados.0.smithi110.stdout:3248: finishing write tid 1 to smithi11075562-22 2024-09-14T03:48:46.933 INFO:tasks.rados.rados.0.smithi110.stdout:3248: finishing write tid 2 to smithi11075562-22 2024-09-14T03:48:46.934 INFO:tasks.rados.rados.0.smithi110.stdout:3246: done (3 left) 2024-09-14T03:48:46.934 INFO:tasks.rados.rados.0.smithi110.stdout:3247: done (2 left) 2024-09-14T03:48:46.934 INFO:tasks.rados.rados.0.smithi110.stdout:3249: done (1 left) 2024-09-14T03:48:46.934 INFO:tasks.rados.rados.0.smithi110.stdout:3250: read oid 18 snap -1 2024-09-14T03:48:46.934 INFO:tasks.rados.rados.0.smithi110.stdout:3250: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:46.934 INFO:tasks.rados.rados.0.smithi110.stdout:3251: read oid 34 snap -1 2024-09-14T03:48:46.934 INFO:tasks.rados.rados.0.smithi110.stdout:3251: expect (ObjNum 993 snap 320 seq_num 993) 2024-09-14T03:48:46.934 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 323 2024-09-14T03:48:46.935 INFO:tasks.rados.rados.0.smithi110.stdout:3252: seq_num 1003 ranges {0=802816,802816=499712,1302528=737280,2039808=368640} 2024-09-14T03:48:46.939 INFO:tasks.rados.rados.0.smithi110.stdout:3252: writing smithi11075562-14 from 0 to 802816 tid 1 2024-09-14T03:48:46.942 INFO:tasks.rados.rados.0.smithi110.stdout:3252: writing smithi11075562-14 from 802816 to 1302528 tid 2 2024-09-14T03:48:46.947 INFO:tasks.rados.rados.0.smithi110.stdout:3252: writing smithi11075562-14 from 1302528 to 2039808 tid 3 2024-09-14T03:48:46.949 INFO:tasks.rados.rados.0.smithi110.stdout:3252: writing smithi11075562-14 from 2039808 to 2408448 tid 4 2024-09-14T03:48:46.949 INFO:tasks.rados.rados.0.smithi110.stdout:3253: copy_from oid 31 from oid 38 current snap is 323 2024-09-14T03:48:46.949 INFO:tasks.rados.rados.0.smithi110.stdout:3250: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:46.953 INFO:tasks.rados.rados.0.smithi110.stdout:3248: finishing write tid 3 to smithi11075562-22 2024-09-14T03:48:46.953 INFO:tasks.rados.rados.0.smithi110.stdout:3248: finishing write tid 4 to smithi11075562-22 2024-09-14T03:48:46.954 INFO:tasks.rados.rados.0.smithi110.stdout:3248: finishing write tid 5 to smithi11075562-22 2024-09-14T03:48:46.954 INFO:tasks.rados.rados.0.smithi110.stdout:3248: finishing write tid 6 to smithi11075562-22 2024-09-14T03:48:46.954 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2032 (ObjNum 1001 snap 323 seq_num 1001) dirty exists 2024-09-14T03:48:46.954 INFO:tasks.rados.rados.0.smithi110.stdout:3248: left oid 22 (ObjNum 1001 snap 323 seq_num 1001) 2024-09-14T03:48:46.954 INFO:tasks.rados.rados.0.smithi110.stdout:3252: finishing write tid 1 to smithi11075562-14 2024-09-14T03:48:46.954 INFO:tasks.rados.rados.0.smithi110.stdout:3252: finishing write tid 2 to smithi11075562-14 2024-09-14T03:48:46.954 INFO:tasks.rados.rados.0.smithi110.stdout:3251: expect (ObjNum 993 snap 320 seq_num 993) 2024-09-14T03:48:47.046 INFO:tasks.rados.rados.0.smithi110.stdout:3252: finishing write tid 3 to smithi11075562-14 2024-09-14T03:48:47.046 INFO:tasks.rados.rados.0.smithi110.stdout:3252: finishing write tid 4 to smithi11075562-14 2024-09-14T03:48:47.046 INFO:tasks.rados.rados.0.smithi110.stdout:3252: finishing write tid 5 to smithi11075562-14 2024-09-14T03:48:47.046 INFO:tasks.rados.rados.0.smithi110.stdout:3252: finishing write tid 6 to smithi11075562-14 2024-09-14T03:48:47.046 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 1906 (ObjNum 1003 snap 323 seq_num 1003) dirty exists 2024-09-14T03:48:47.046 INFO:tasks.rados.rados.0.smithi110.stdout:3252: left oid 14 (ObjNum 1003 snap 323 seq_num 1003) 2024-09-14T03:48:47.046 INFO:tasks.rados.rados.0.smithi110.stdout:3253: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:48:47.047 INFO:tasks.rados.rados.0.smithi110.stdout:3253: finishing copy_from to smithi11075562-31 2024-09-14T03:48:47.047 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 1622 (ObjNum 963 snap 310 seq_num 963) dirty exists 2024-09-14T03:48:47.047 INFO:tasks.rados.rados.0.smithi110.stdout:3248: done (4 left) 2024-09-14T03:48:47.047 INFO:tasks.rados.rados.0.smithi110.stdout:3250: done (3 left) 2024-09-14T03:48:47.047 INFO:tasks.rados.rados.0.smithi110.stdout:3251: done (2 left) 2024-09-14T03:48:47.047 INFO:tasks.rados.rados.0.smithi110.stdout:3252: done (1 left) 2024-09-14T03:48:47.047 INFO:tasks.rados.rados.0.smithi110.stdout:3253: done (0 left) 2024-09-14T03:48:47.047 INFO:tasks.rados.rados.0.smithi110.stdout:3254: setattr oid 41 current snap is 323 2024-09-14T03:48:47.048 INFO:tasks.rados.rados.0.smithi110.stdout:3255: read oid 5 snap 321 2024-09-14T03:48:47.048 INFO:tasks.rados.rados.0.smithi110.stdout:3255: expect (ObjNum 936 snap 303 seq_num 936) 2024-09-14T03:48:47.048 INFO:tasks.rados.rados.0.smithi110.stdout:3256: snap_remove snap 320 2024-09-14T03:48:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:46 smithi110 ceph-mon[88262]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T03:48:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:46 smithi110 ceph-mon[86423]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T03:48:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:46 smithi195 ceph-mon[82388]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T03:48:47.820 INFO:tasks.rados.rados.0.smithi110.stdout:3256: done (2 left) 2024-09-14T03:48:47.820 INFO:tasks.rados.rados.0.smithi110.stdout:3257: snap_remove snap 318 2024-09-14T03:48:47.820 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1907 (ObjNum 66134064 snap 21934 seq_num 304) dirty exists 2024-09-14T03:48:47.820 INFO:tasks.rados.rados.0.smithi110.stdout:3255: expect (ObjNum 936 snap 303 seq_num 936) 2024-09-14T03:48:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:47 smithi110 ceph-mon[86423]: pgmap v711: 121 pgs: 121 active+clean; 199 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 11 op/s 2024-09-14T03:48:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:47 smithi110 ceph-mon[86423]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T03:48:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:47 smithi110 ceph-mon[88262]: pgmap v711: 121 pgs: 121 active+clean; 199 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 11 op/s 2024-09-14T03:48:48.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:47 smithi110 ceph-mon[88262]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T03:48:48.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:47 smithi195 ceph-mon[82388]: pgmap v711: 121 pgs: 121 active+clean; 199 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 11 op/s 2024-09-14T03:48:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:47 smithi195 ceph-mon[82388]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T03:48:48.797 INFO:tasks.rados.rados.0.smithi110.stdout:3254: done (2 left) 2024-09-14T03:48:48.797 INFO:tasks.rados.rados.0.smithi110.stdout:3255: done (1 left) 2024-09-14T03:48:48.797 INFO:tasks.rados.rados.0.smithi110.stdout:3257: done (0 left) 2024-09-14T03:48:48.797 INFO:tasks.rados.rados.0.smithi110.stdout:3258: copy_from oid 5 from oid 40 current snap is 323 2024-09-14T03:48:48.797 INFO:tasks.rados.rados.0.smithi110.stdout:3259: setattr oid 39 current snap is 323 2024-09-14T03:48:48.800 INFO:tasks.rados.rados.0.smithi110.stdout:3260: snap_remove snap 315 2024-09-14T03:48:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:48 smithi110 ceph-mon[88262]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T03:48:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:48 smithi110 ceph-mon[88262]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T03:48:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:48 smithi110 ceph-mon[86423]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T03:48:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:48 smithi110 ceph-mon[86423]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T03:48:49.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:48 smithi195 ceph-mon[82388]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T03:48:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:48 smithi195 ceph-mon[82388]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T03:48:49.799 INFO:tasks.rados.rados.0.smithi110.stdout:3260: done (2 left) 2024-09-14T03:48:49.799 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 323 2024-09-14T03:48:49.799 INFO:tasks.rados.rados.0.smithi110.stdout:3261: seq_num 1004 ranges {2359296=16384} 2024-09-14T03:48:49.813 INFO:tasks.rados.rados.0.smithi110.stdout:3261: writing smithi11075562-42 from 2359296 to 2375680 tid 1 2024-09-14T03:48:49.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 1908 (ObjNum 970 snap 312 seq_num 970) dirty exists 2024-09-14T03:48:49.813 INFO:tasks.rados.rados.0.smithi110.stdout:3258: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:48:49.813 INFO:tasks.rados.rados.0.smithi110.stdout:3258: finishing copy_from to smithi11075562-5 2024-09-14T03:48:49.813 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2001 (ObjNum 962 snap 310 seq_num 962) dirty exists 2024-09-14T03:48:49.814 INFO:tasks.rados.rados.0.smithi110.stdout:3258: done (2 left) 2024-09-14T03:48:49.814 INFO:tasks.rados.rados.0.smithi110.stdout:3259: done (1 left) 2024-09-14T03:48:49.814 INFO:tasks.rados.rados.0.smithi110.stdout:3262: rollback oid 28 current snap is 323 2024-09-14T03:48:49.814 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 28 to 316 2024-09-14T03:48:49.814 INFO:tasks.rados.rados.0.smithi110.stdout:3263: snap_remove snap 317 2024-09-14T03:48:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:49 smithi110 ceph-mon[88262]: pgmap v714: 121 pgs: 121 active+clean; 209 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 7.1 MiB/s wr, 28 op/s 2024-09-14T03:48:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:49 smithi110 ceph-mon[88262]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T03:48:50.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:49 smithi110 ceph-mon[86423]: pgmap v714: 121 pgs: 121 active+clean; 209 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 7.1 MiB/s wr, 28 op/s 2024-09-14T03:48:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:49 smithi110 ceph-mon[86423]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T03:48:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:49 smithi195 ceph-mon[82388]: pgmap v714: 121 pgs: 121 active+clean; 209 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 7.1 MiB/s wr, 28 op/s 2024-09-14T03:48:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:49 smithi195 ceph-mon[82388]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T03:48:50.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:48:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:48:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:48:50.800 INFO:tasks.rados.rados.0.smithi110.stdout:3261: finishing write tid 1 to smithi11075562-42 2024-09-14T03:48:50.800 INFO:tasks.rados.rados.0.smithi110.stdout:3261: finishing write tid 2 to smithi11075562-42 2024-09-14T03:48:50.800 INFO:tasks.rados.rados.0.smithi110.stdout:3262: finishing rollback tid 0 to smithi11075562-28 2024-09-14T03:48:50.800 INFO:tasks.rados.rados.0.smithi110.stdout:3262: finishing rollback tid 1 to smithi11075562-28 2024-09-14T03:48:50.800 INFO:tasks.rados.rados.0.smithi110.stdout:3262: finishing rollback tid 2 to smithi11075562-28 2024-09-14T03:48:50.800 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 1999 (ObjNum 969 snap 311 seq_num 969) dirty exists 2024-09-14T03:48:50.801 INFO:tasks.rados.rados.0.smithi110.stdout:3261: finishing write tid 3 to smithi11075562-42 2024-09-14T03:48:50.801 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2040 (ObjNum 1004 snap 323 seq_num 1004) dirty exists 2024-09-14T03:48:50.801 INFO:tasks.rados.rados.0.smithi110.stdout:3261: left oid 42 (ObjNum 1004 snap 323 seq_num 1004) 2024-09-14T03:48:50.801 INFO:tasks.rados.rados.0.smithi110.stdout:3261: done (2 left) 2024-09-14T03:48:50.801 INFO:tasks.rados.rados.0.smithi110.stdout:3262: done (1 left) 2024-09-14T03:48:50.801 INFO:tasks.rados.rados.0.smithi110.stdout:3263: done (0 left) 2024-09-14T03:48:50.801 INFO:tasks.rados.rados.0.smithi110.stdout:3264: delete oid 6 current snap is 323 2024-09-14T03:48:50.810 INFO:tasks.rados.rados.0.smithi110.stdout:3264: done (0 left) 2024-09-14T03:48:50.810 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 323 2024-09-14T03:48:50.810 INFO:tasks.rados.rados.0.smithi110.stdout:3265: seq_num 1006 ranges {0=49152} 2024-09-14T03:48:50.810 INFO:tasks.rados.rados.0.smithi110.stdout:3265: writing smithi11075562-41 from 0 to 49152 tid 1 2024-09-14T03:48:50.810 INFO:tasks.rados.rados.0.smithi110.stdout:3266: delete oid 48 current snap is 323 2024-09-14T03:48:50.819 INFO:tasks.rados.rados.0.smithi110.stdout:3266: done (1 left) 2024-09-14T03:48:50.819 INFO:tasks.rados.rados.0.smithi110.stdout:3267: copy_from oid 34 from oid 3 current snap is 323 2024-09-14T03:48:50.819 INFO:tasks.rados.rados.0.smithi110.stdout:3268: copy_from oid 25 from oid 31 current snap is 323 2024-09-14T03:48:50.819 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 323 2024-09-14T03:48:50.819 INFO:tasks.rados.rados.0.smithi110.stdout:3269: seq_num 1008 ranges {8159232=90112} 2024-09-14T03:48:50.864 INFO:tasks.rados.rados.0.smithi110.stdout:3269: writing smithi11075562-4 from 8159232 to 8249344 tid 1 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3270: rollback oid 12 current snap is 323 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 321 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3271: delete oid 2 current snap is 323 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3265: finishing write tid 1 to smithi11075562-41 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3265: finishing write tid 2 to smithi11075562-41 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3265: finishing write tid 3 to smithi11075562-41 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1910 (ObjNum 1006 snap 323 seq_num 1006) dirty exists 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3265: left oid 41 (ObjNum 1006 snap 323 seq_num 1006) 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3267: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3268: finishing copy_from racing read to smithi11075562-25 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3267: finishing copy_from to smithi11075562-34 2024-09-14T03:48:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1683 (ObjNum 965 snap 311 seq_num 965) dirty exists 2024-09-14T03:48:50.866 INFO:tasks.rados.rados.0.smithi110.stdout:3268: finishing copy_from to smithi11075562-25 2024-09-14T03:48:50.866 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 1622 (ObjNum 963 snap 310 seq_num 963) dirty exists 2024-09-14T03:48:50.866 INFO:tasks.rados.rados.0.smithi110.stdout:3270: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:48:50.867 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2289 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:48:50.867 INFO:tasks.rados.rados.0.smithi110.stdout:3269: finishing write tid 1 to smithi11075562-4 2024-09-14T03:48:50.867 INFO:tasks.rados.rados.0.smithi110.stdout:3269: finishing write tid 2 to smithi11075562-4 2024-09-14T03:48:50.868 INFO:tasks.rados.rados.0.smithi110.stdout:3265: done (5 left) 2024-09-14T03:48:50.868 INFO:tasks.rados.rados.0.smithi110.stdout:3267: done (4 left) 2024-09-14T03:48:50.868 INFO:tasks.rados.rados.0.smithi110.stdout:3268: done (3 left) 2024-09-14T03:48:50.869 INFO:tasks.rados.rados.0.smithi110.stdout:3270: done (2 left) 2024-09-14T03:48:50.869 INFO:tasks.rados.rados.0.smithi110.stdout:3271: done (1 left) 2024-09-14T03:48:50.869 INFO:tasks.rados.rados.0.smithi110.stdout:3272: copy_from oid 37 from oid 22 current snap is 323 2024-09-14T03:48:50.869 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 323 2024-09-14T03:48:50.869 INFO:tasks.rados.rados.0.smithi110.stdout:3273: seq_num 1010 ranges {5013504=122880} 2024-09-14T03:48:50.897 INFO:tasks.rados.rados.0.smithi110.stdout:3273: writing smithi11075562-36 from 5013504 to 5136384 tid 1 2024-09-14T03:48:50.897 INFO:tasks.rados.rados.0.smithi110.stdout:3274: rollback oid 24 current snap is 323 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 316 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:3269: finishing write tid 3 to smithi11075562-4 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2143 (ObjNum 1008 snap 323 seq_num 1008) dirty exists 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:3269: left oid 4 (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:3272: finishing copy_from racing read to smithi11075562-37 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:3272: finishing copy_from to smithi11075562-37 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 2032 (ObjNum 1001 snap 323 seq_num 1001) dirty exists 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:3269: done (3 left) 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:3272: done (2 left) 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:3275: rollback oid 32 current snap is 323 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 319 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 323 2024-09-14T03:48:50.898 INFO:tasks.rados.rados.0.smithi110.stdout:3276: seq_num 1011 ranges {0=139264} 2024-09-14T03:48:50.899 INFO:tasks.rados.rados.0.smithi110.stdout:3276: writing smithi11075562-48 from 0 to 139264 tid 1 2024-09-14T03:48:50.899 INFO:tasks.rados.rados.0.smithi110.stdout:3277: delete oid 26 current snap is 323 2024-09-14T03:48:50.901 INFO:tasks.rados.rados.0.smithi110.stdout:3275: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:48:50.902 INFO:tasks.rados.rados.0.smithi110.stdout:3276: finishing write tid 1 to smithi11075562-48 2024-09-14T03:48:50.902 INFO:tasks.rados.rados.0.smithi110.stdout:3275: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:48:50.902 INFO:tasks.rados.rados.0.smithi110.stdout:3274: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:48:50.902 INFO:tasks.rados.rados.0.smithi110.stdout:3273: finishing write tid 1 to smithi11075562-36 2024-09-14T03:48:50.902 INFO:tasks.rados.rados.0.smithi110.stdout:3276: finishing write tid 2 to smithi11075562-48 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:3275: finishing rollback tid 2 to smithi11075562-32 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2035 (ObjNum 990 snap 318 seq_num 990) dirty exists 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:3274: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:3275: done (4 left) 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:3277: done (3 left) 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:3278: read oid 35 snap -1 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:3278: expect deleted 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:3279: delete oid 8 current snap is 323 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:3274: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:48:50.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2005 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:48:50.905 INFO:tasks.rados.rados.0.smithi110.stdout:3276: finishing write tid 3 to smithi11075562-48 2024-09-14T03:48:50.905 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1994 (ObjNum 1011 snap 323 seq_num 1011) dirty exists 2024-09-14T03:48:50.905 INFO:tasks.rados.rados.0.smithi110.stdout:3276: left oid 48 (ObjNum 1011 snap 323 seq_num 1011) 2024-09-14T03:48:50.905 INFO:tasks.rados.rados.0.smithi110.stdout:3273: finishing write tid 2 to smithi11075562-36 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:3274: done (4 left) 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:3276: done (3 left) 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:3278: done (2 left) 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:3279: done (1 left) 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:3280: copy_from oid 47 from oid 37 current snap is 323 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:3273: finishing write tid 3 to smithi11075562-36 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2311 (ObjNum 1010 snap 323 seq_num 1010) dirty exists 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:3273: left oid 36 (ObjNum 1010 snap 323 seq_num 1010) 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:3273: done (1 left) 2024-09-14T03:48:50.907 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 323 2024-09-14T03:48:50.908 INFO:tasks.rados.rados.0.smithi110.stdout:3281: seq_num 1014 ranges {1785856=188416} 2024-09-14T03:48:50.917 INFO:tasks.rados.rados.0.smithi110.stdout:3281: writing smithi11075562-1 from 1785856 to 1974272 tid 1 2024-09-14T03:48:50.917 INFO:tasks.rados.rados.0.smithi110.stdout:3282: read oid 25 snap -1 2024-09-14T03:48:50.917 INFO:tasks.rados.rados.0.smithi110.stdout:3282: expect (ObjNum 963 snap 310 seq_num 963) 2024-09-14T03:48:50.917 INFO:tasks.rados.rados.0.smithi110.stdout:3283: delete oid 31 current snap is 323 2024-09-14T03:48:50.917 INFO:tasks.rados.rados.0.smithi110.stdout:3280: finishing copy_from racing read to smithi11075562-47 2024-09-14T03:48:50.917 INFO:tasks.rados.rados.0.smithi110.stdout:3280: finishing copy_from to smithi11075562-47 2024-09-14T03:48:50.917 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 2032 (ObjNum 1001 snap 323 seq_num 1001) dirty exists 2024-09-14T03:48:50.919 INFO:tasks.rados.rados.0.smithi110.stdout:3280: done (3 left) 2024-09-14T03:48:50.919 INFO:tasks.rados.rados.0.smithi110.stdout:3283: done (2 left) 2024-09-14T03:48:50.919 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 323 2024-09-14T03:48:50.919 INFO:tasks.rados.rados.0.smithi110.stdout:3284: seq_num 1016 ranges {0=221184} 2024-09-14T03:48:50.920 INFO:tasks.rados.rados.0.smithi110.stdout:3284: writing smithi11075562-6 from 0 to 221184 tid 1 2024-09-14T03:48:50.920 INFO:tasks.rados.rados.0.smithi110.stdout:3285: rollback oid 27 current snap is 323 2024-09-14T03:48:50.920 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 314 2024-09-14T03:48:50.920 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 323 2024-09-14T03:48:50.920 INFO:tasks.rados.rados.0.smithi110.stdout:3286: seq_num 1017 ranges {9265152=237568} 2024-09-14T03:48:50.971 INFO:tasks.rados.rados.0.smithi110.stdout:3286: writing smithi11075562-40 from 9265152 to 9502720 tid 1 2024-09-14T03:48:50.971 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 323 2024-09-14T03:48:50.971 INFO:tasks.rados.rados.0.smithi110.stdout:3281: finishing write tid 1 to smithi11075562-1 2024-09-14T03:48:50.971 INFO:tasks.rados.rados.0.smithi110.stdout:3287: seq_num 1018 ranges {3866624=253952} 2024-09-14T03:48:50.992 INFO:tasks.rados.rados.0.smithi110.stdout:3287: writing smithi11075562-7 from 3866624 to 4120576 tid 1 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3288: snap_create 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3281: finishing write tid 2 to smithi11075562-1 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3285: finishing rollback tid 0 to smithi11075562-27 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3285: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1812 (ObjNum 33 snap 0 seq_num 1494394119) dirty dne 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3281: finishing write tid 3 to smithi11075562-1 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2047 (ObjNum 1014 snap 323 seq_num 1014) dirty exists 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3281: left oid 1 (ObjNum 1014 snap 323 seq_num 1014) 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3284: finishing write tid 1 to smithi11075562-6 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3284: finishing write tid 2 to smithi11075562-6 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3284: finishing write tid 3 to smithi11075562-6 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2130 (ObjNum 1016 snap 323 seq_num 1016) dirty exists 2024-09-14T03:48:50.993 INFO:tasks.rados.rados.0.smithi110.stdout:3284: left oid 6 (ObjNum 1016 snap 323 seq_num 1016) 2024-09-14T03:48:50.994 INFO:tasks.rados.rados.0.smithi110.stdout:3282: expect (ObjNum 963 snap 310 seq_num 963) 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:3286: finishing write tid 1 to smithi11075562-40 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:3286: finishing write tid 2 to smithi11075562-40 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:3286: finishing write tid 3 to smithi11075562-40 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 2132 (ObjNum 1017 snap 323 seq_num 1017) dirty exists 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:3286: left oid 40 (ObjNum 1017 snap 323 seq_num 1017) 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:3287: finishing write tid 1 to smithi11075562-7 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:3287: finishing write tid 2 to smithi11075562-7 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:3287: finishing write tid 3 to smithi11075562-7 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1628 (ObjNum 1018 snap 323 seq_num 1018) dirty exists 2024-09-14T03:48:51.037 INFO:tasks.rados.rados.0.smithi110.stdout:3287: left oid 7 (ObjNum 1018 snap 323 seq_num 1018) 2024-09-14T03:48:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:3281: done (6 left) 2024-09-14T03:48:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:3282: done (5 left) 2024-09-14T03:48:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:3284: done (4 left) 2024-09-14T03:48:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:3285: done (3 left) 2024-09-14T03:48:51.829 INFO:tasks.rados.rados.0.smithi110.stdout:3286: done (2 left) 2024-09-14T03:48:51.830 INFO:tasks.rados.rados.0.smithi110.stdout:3287: done (1 left) 2024-09-14T03:48:51.830 INFO:tasks.rados.rados.0.smithi110.stdout:3288: done (0 left) 2024-09-14T03:48:51.830 INFO:tasks.rados.rados.0.smithi110.stdout:3289: delete oid 47 current snap is 324 2024-09-14T03:48:51.835 INFO:tasks.rados.rados.0.smithi110.stdout:3289: done (0 left) 2024-09-14T03:48:51.836 INFO:tasks.rados.rados.0.smithi110.stdout:3290: snap_create 2024-09-14T03:48:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:51 smithi110 ceph-mon[88262]: pgmap v717: 121 pgs: 121 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.8 MiB/s wr, 29 op/s 2024-09-14T03:48:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:51 smithi110 ceph-mon[88262]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T03:48:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:48:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:48:51] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-14T03:48:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:51 smithi110 ceph-mon[86423]: pgmap v717: 121 pgs: 121 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.8 MiB/s wr, 29 op/s 2024-09-14T03:48:51.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:51 smithi110 ceph-mon[86423]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T03:48:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:51 smithi195 ceph-mon[82388]: pgmap v717: 121 pgs: 121 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.8 MiB/s wr, 29 op/s 2024-09-14T03:48:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:51 smithi195 ceph-mon[82388]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T03:48:52.835 INFO:tasks.rados.rados.0.smithi110.stdout:3290: done (0 left) 2024-09-14T03:48:52.835 INFO:tasks.rados.rados.0.smithi110.stdout:3291: read oid 39 snap -1 2024-09-14T03:48:52.835 INFO:tasks.rados.rados.0.smithi110.stdout:3291: expect (ObjNum 970 snap 312 seq_num 970) 2024-09-14T03:48:52.835 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 325 2024-09-14T03:48:52.836 INFO:tasks.rados.rados.0.smithi110.stdout:3292: seq_num 1020 ranges {9265152=286720} 2024-09-14T03:48:52.894 INFO:tasks.rados.rados.0.smithi110.stdout:3292: writing smithi11075562-5 from 9265152 to 9551872 tid 1 2024-09-14T03:48:52.894 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 325 2024-09-14T03:48:52.895 INFO:tasks.rados.rados.0.smithi110.stdout:3293: seq_num 1021 ranges {139264=303104} 2024-09-14T03:48:52.897 INFO:tasks.rados.rados.0.smithi110.stdout:3293: writing smithi11075562-48 from 139264 to 442368 tid 1 2024-09-14T03:48:52.897 INFO:tasks.rados.rados.0.smithi110.stdout:3294: rollback oid 41 current snap is 325 2024-09-14T03:48:52.897 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 322 2024-09-14T03:48:52.897 INFO:tasks.rados.rados.0.smithi110.stdout:3295: delete oid 33 current snap is 325 2024-09-14T03:48:52.904 INFO:tasks.rados.rados.0.smithi110.stdout:3293: finishing write tid 1 to smithi11075562-48 2024-09-14T03:48:52.904 INFO:tasks.rados.rados.0.smithi110.stdout:3293: finishing write tid 2 to smithi11075562-48 2024-09-14T03:48:52.904 INFO:tasks.rados.rados.0.smithi110.stdout:3295: done (4 left) 2024-09-14T03:48:52.904 INFO:tasks.rados.rados.0.smithi110.stdout:3296: delete oid 29 current snap is 325 2024-09-14T03:48:52.905 INFO:tasks.rados.rados.0.smithi110.stdout:3292: finishing write tid 1 to smithi11075562-5 2024-09-14T03:48:52.905 INFO:tasks.rados.rados.0.smithi110.stdout:3292: finishing write tid 2 to smithi11075562-5 2024-09-14T03:48:52.905 INFO:tasks.rados.rados.0.smithi110.stdout:3293: finishing write tid 3 to smithi11075562-48 2024-09-14T03:48:52.905 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1996 (ObjNum 1021 snap 325 seq_num 1021) dirty exists 2024-09-14T03:48:52.905 INFO:tasks.rados.rados.0.smithi110.stdout:3293: left oid 48 (ObjNum 1021 snap 325 seq_num 1021) 2024-09-14T03:48:52.907 INFO:tasks.rados.rados.0.smithi110.stdout:3292: finishing write tid 3 to smithi11075562-5 2024-09-14T03:48:52.907 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2318 (ObjNum 1020 snap 325 seq_num 1020) dirty exists 2024-09-14T03:48:52.907 INFO:tasks.rados.rados.0.smithi110.stdout:3292: left oid 5 (ObjNum 1020 snap 325 seq_num 1020) 2024-09-14T03:48:52.908 INFO:tasks.rados.rados.0.smithi110.stdout:3292: done (4 left) 2024-09-14T03:48:52.908 INFO:tasks.rados.rados.0.smithi110.stdout:3293: done (3 left) 2024-09-14T03:48:52.908 INFO:tasks.rados.rados.0.smithi110.stdout:3296: done (2 left) 2024-09-14T03:48:52.908 INFO:tasks.rados.rados.0.smithi110.stdout:3297: read oid 32 snap 322 2024-09-14T03:48:52.908 INFO:tasks.rados.rados.0.smithi110.stdout:3297: expect (ObjNum 996 snap 322 seq_num 996) 2024-09-14T03:48:52.908 INFO:tasks.rados.rados.0.smithi110.stdout:3298: snap_remove snap 319 2024-09-14T03:48:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:52 smithi110 ceph-mon[88262]: pgmap v720: 121 pgs: 121 active+clean; 251 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 9.2 MiB/s wr, 19 op/s 2024-09-14T03:48:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:52 smithi110 ceph-mon[88262]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T03:48:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:52 smithi110 ceph-mon[86423]: pgmap v720: 121 pgs: 121 active+clean; 251 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 9.2 MiB/s wr, 19 op/s 2024-09-14T03:48:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:52 smithi110 ceph-mon[86423]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T03:48:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:52 smithi195 ceph-mon[82388]: pgmap v720: 121 pgs: 121 active+clean; 251 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 9.2 MiB/s wr, 19 op/s 2024-09-14T03:48:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:52 smithi195 ceph-mon[82388]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T03:48:53.840 INFO:tasks.rados.rados.0.smithi110.stdout:3291: expect (ObjNum 970 snap 312 seq_num 970) 2024-09-14T03:48:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:53 smithi110 ceph-mon[88262]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T03:48:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:53 smithi110 ceph-mon[86423]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T03:48:54.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:53 smithi195 ceph-mon[82388]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T03:48:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:54 smithi110 ceph-mon[86423]: pgmap v722: 121 pgs: 121 active+clean; 251 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.5 MiB/s wr, 21 op/s 2024-09-14T03:48:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:54 smithi110 ceph-mon[86423]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T03:48:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:54 smithi110 ceph-mon[88262]: pgmap v722: 121 pgs: 121 active+clean; 251 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.5 MiB/s wr, 21 op/s 2024-09-14T03:48:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:54 smithi110 ceph-mon[88262]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T03:48:55.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:54 smithi195 ceph-mon[82388]: pgmap v722: 121 pgs: 121 active+clean; 251 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.5 MiB/s wr, 21 op/s 2024-09-14T03:48:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:54 smithi195 ceph-mon[82388]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T03:48:55.814 INFO:tasks.rados.rados.0.smithi110.stdout:3294: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:48:55.814 INFO:tasks.rados.rados.0.smithi110.stdout:3294: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:48:55.814 INFO:tasks.rados.rados.0.smithi110.stdout:3294: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:48:55.814 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1913 (ObjNum 33 snap 0 seq_num 1801941607) dirty exists 2024-09-14T03:48:55.814 INFO:tasks.rados.rados.0.smithi110.stdout:3297: expect (ObjNum 996 snap 322 seq_num 996) 2024-09-14T03:48:55.962 INFO:tasks.rados.rados.0.smithi110.stdout:3291: done (3 left) 2024-09-14T03:48:55.962 INFO:tasks.rados.rados.0.smithi110.stdout:3294: done (2 left) 2024-09-14T03:48:55.962 INFO:tasks.rados.rados.0.smithi110.stdout:3297: done (1 left) 2024-09-14T03:48:55.962 INFO:tasks.rados.rados.0.smithi110.stdout:3298: done (0 left) 2024-09-14T03:48:55.963 INFO:tasks.rados.rados.0.smithi110.stdout:3299: delete oid 22 current snap is 325 2024-09-14T03:48:55.965 INFO:tasks.rados.rados.0.smithi110.stdout:3299: done (0 left) 2024-09-14T03:48:55.965 INFO:tasks.rados.rados.0.smithi110.stdout:3300: delete oid 22 current snap is 325 2024-09-14T03:48:55.967 INFO:tasks.rados.rados.0.smithi110.stdout:3300: done (0 left) 2024-09-14T03:48:55.968 INFO:tasks.rados.rados.0.smithi110.stdout:3301: rollback oid 41 current snap is 325 2024-09-14T03:48:55.968 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 316 2024-09-14T03:48:55.968 INFO:tasks.rados.rados.0.smithi110.stdout:3302: rollback oid 19 current snap is 325 2024-09-14T03:48:55.968 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 324 2024-09-14T03:48:55.968 INFO:tasks.rados.rados.0.smithi110.stdout:3303: rollback oid 24 current snap is 325 2024-09-14T03:48:55.968 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 321 2024-09-14T03:48:55.968 INFO:tasks.rados.rados.0.smithi110.stdout:3304: snap_create 2024-09-14T03:48:55.971 INFO:tasks.rados.rados.0.smithi110.stdout:3301: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:48:55.972 INFO:tasks.rados.rados.0.smithi110.stdout:3301: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:48:55.972 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1915 (ObjNum 33 snap 0 seq_num 1494479554) dirty dne 2024-09-14T03:48:55.972 INFO:tasks.rados.rados.0.smithi110.stdout:3303: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:48:55.973 INFO:tasks.rados.rados.0.smithi110.stdout:3302: finishing rollback tid 0 to smithi11075562-19 2024-09-14T03:48:55.973 INFO:tasks.rados.rados.0.smithi110.stdout:3303: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:48:55.974 INFO:tasks.rados.rados.0.smithi110.stdout:3302: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:48:55.974 INFO:tasks.rados.rados.0.smithi110.stdout:3302: finishing rollback tid 2 to smithi11075562-19 2024-09-14T03:48:55.974 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2135 (ObjNum 949 snap 307 seq_num 949) dirty exists 2024-09-14T03:48:55.974 INFO:tasks.rados.rados.0.smithi110.stdout:3303: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:48:55.974 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2008 (ObjNum 987 snap 318 seq_num 987) dirty exists 2024-09-14T03:48:56.850 INFO:tasks.rados.rados.0.smithi110.stdout:3301: done (3 left) 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3302: done (2 left) 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3303: done (1 left) 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3304: done (0 left) 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3305: read oid 49 snap -1 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3305: expect (ObjNum 930 snap 302 seq_num 930) 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3306: read oid 7 snap -1 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3306: expect (ObjNum 1018 snap 323 seq_num 1018) 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 326 2024-09-14T03:48:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3307: seq_num 1026 ranges {1736704=385024} 2024-09-14T03:48:56.863 INFO:tasks.rados.rados.0.smithi110.stdout:3307: writing smithi11075562-25 from 1736704 to 2121728 tid 1 2024-09-14T03:48:56.863 INFO:tasks.rados.rados.0.smithi110.stdout:3308: snap_create 2024-09-14T03:48:56.863 INFO:tasks.rados.rados.0.smithi110.stdout:3305: expect (ObjNum 930 snap 302 seq_num 930) 2024-09-14T03:48:56.928 INFO:tasks.rados.rados.0.smithi110.stdout:3306: expect (ObjNum 1018 snap 323 seq_num 1018) 2024-09-14T03:48:57.132 INFO:tasks.rados.rados.0.smithi110.stdout:3307: finishing write tid 1 to smithi11075562-25 2024-09-14T03:48:57.132 INFO:tasks.rados.rados.0.smithi110.stdout:3307: finishing write tid 2 to smithi11075562-25 2024-09-14T03:48:57.132 INFO:tasks.rados.rados.0.smithi110.stdout:3307: finishing write tid 3 to smithi11075562-25 2024-09-14T03:48:57.132 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2329 (ObjNum 1026 snap 326 seq_num 1026) dirty exists 2024-09-14T03:48:57.132 INFO:tasks.rados.rados.0.smithi110.stdout:3307: left oid 25 (ObjNum 1026 snap 326 seq_num 1026) 2024-09-14T03:48:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:56 smithi110 ceph-mon[86423]: pgmap v724: 121 pgs: 121 active+clean; 256 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-14T03:48:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:56 smithi110 ceph-mon[88262]: pgmap v724: 121 pgs: 121 active+clean; 256 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-14T03:48:57.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:56 smithi195 ceph-mon[82388]: pgmap v724: 121 pgs: 121 active+clean; 256 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-14T03:48:57.856 INFO:tasks.rados.rados.0.smithi110.stdout:3305: done (3 left) 2024-09-14T03:48:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:3306: done (2 left) 2024-09-14T03:48:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:3307: done (1 left) 2024-09-14T03:48:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:3308: done (0 left) 2024-09-14T03:48:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:3309: read oid 36 snap 316 2024-09-14T03:48:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:3309: expect (ObjNum 975 snap 314 seq_num 975) 2024-09-14T03:48:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 327 2024-09-14T03:48:57.857 INFO:tasks.rados.rados.0.smithi110.stdout:3310: seq_num 1027 ranges {1785856=409600} 2024-09-14T03:48:57.868 INFO:tasks.rados.rados.0.smithi110.stdout:3310: writing smithi11075562-15 from 1785856 to 2195456 tid 1 2024-09-14T03:48:57.868 INFO:tasks.rados.rados.0.smithi110.stdout:3311: setattr oid 33 current snap is 327 2024-09-14T03:48:57.874 INFO:tasks.rados.rados.0.smithi110.stdout:3310: finishing write tid 1 to smithi11075562-15 2024-09-14T03:48:57.874 INFO:tasks.rados.rados.0.smithi110.stdout:3310: finishing write tid 2 to smithi11075562-15 2024-09-14T03:48:57.874 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 327 2024-09-14T03:48:57.875 INFO:tasks.rados.rados.0.smithi110.stdout:3312: seq_num 1028 ranges {4120576=425984} 2024-09-14T03:48:57.900 INFO:tasks.rados.rados.0.smithi110.stdout:3312: writing smithi11075562-7 from 4120576 to 4546560 tid 1 2024-09-14T03:48:57.900 INFO:tasks.rados.rados.0.smithi110.stdout:3313: snap_create 2024-09-14T03:48:57.900 INFO:tasks.rados.rados.0.smithi110.stdout:3310: finishing write tid 3 to smithi11075562-15 2024-09-14T03:48:57.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2038 (ObjNum 1027 snap 327 seq_num 1027) dirty exists 2024-09-14T03:48:57.900 INFO:tasks.rados.rados.0.smithi110.stdout:3310: left oid 15 (ObjNum 1027 snap 327 seq_num 1027) 2024-09-14T03:48:57.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 2134 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:48:57.900 INFO:tasks.rados.rados.0.smithi110.stdout:3309: expect (ObjNum 975 snap 314 seq_num 975) 2024-09-14T03:48:58.102 INFO:tasks.rados.rados.0.smithi110.stdout:3312: finishing write tid 1 to smithi11075562-7 2024-09-14T03:48:58.102 INFO:tasks.rados.rados.0.smithi110.stdout:3312: finishing write tid 2 to smithi11075562-7 2024-09-14T03:48:58.102 INFO:tasks.rados.rados.0.smithi110.stdout:3312: finishing write tid 3 to smithi11075562-7 2024-09-14T03:48:58.103 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1630 (ObjNum 1028 snap 327 seq_num 1028) dirty exists 2024-09-14T03:48:58.103 INFO:tasks.rados.rados.0.smithi110.stdout:3312: left oid 7 (ObjNum 1028 snap 327 seq_num 1028) 2024-09-14T03:48:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:57 smithi110 ceph-mon[88262]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T03:48:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:57 smithi110 ceph-mon[86423]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T03:48:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:58.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:57 smithi195 ceph-mon[82388]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T03:48:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:48:58.799 INFO:tasks.rados.rados.0.smithi110.stdout:3309: done (4 left) 2024-09-14T03:48:58.799 INFO:tasks.rados.rados.0.smithi110.stdout:3310: done (3 left) 2024-09-14T03:48:58.799 INFO:tasks.rados.rados.0.smithi110.stdout:3311: done (2 left) 2024-09-14T03:48:58.799 INFO:tasks.rados.rados.0.smithi110.stdout:3312: done (1 left) 2024-09-14T03:48:58.799 INFO:tasks.rados.rados.0.smithi110.stdout:3313: done (0 left) 2024-09-14T03:48:58.799 INFO:tasks.rados.rados.0.smithi110.stdout:3314: delete oid 25 current snap is 328 2024-09-14T03:48:58.803 INFO:tasks.rados.rados.0.smithi110.stdout:3314: done (0 left) 2024-09-14T03:48:58.803 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 328 2024-09-14T03:48:58.803 INFO:tasks.rados.rados.0.smithi110.stdout:3315: seq_num 1030 ranges {491520=458752} 2024-09-14T03:48:58.808 INFO:tasks.rados.rados.0.smithi110.stdout:3315: writing smithi11075562-45 from 491520 to 950272 tid 1 2024-09-14T03:48:58.809 INFO:tasks.rados.rados.0.smithi110.stdout:3316: read oid 17 snap -1 2024-09-14T03:48:58.809 INFO:tasks.rados.rados.0.smithi110.stdout:3316: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:58.809 INFO:tasks.rados.rados.0.smithi110.stdout:3317: delete oid 32 current snap is 328 2024-09-14T03:48:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:3317: done (2 left) 2024-09-14T03:48:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 328 2024-09-14T03:48:58.816 INFO:tasks.rados.rados.0.smithi110.stdout:3318: seq_num 1032 ranges {1974272=491520} 2024-09-14T03:48:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:3318: writing smithi11075562-1 from 1974272 to 2465792 tid 1 2024-09-14T03:48:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:3319: snap_create 2024-09-14T03:48:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:3315: finishing write tid 1 to smithi11075562-45 2024-09-14T03:48:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:3315: finishing write tid 2 to smithi11075562-45 2024-09-14T03:48:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:3315: finishing write tid 3 to smithi11075562-45 2024-09-14T03:48:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2334 (ObjNum 1030 snap 328 seq_num 1030) dirty exists 2024-09-14T03:48:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:3315: left oid 45 (ObjNum 1030 snap 328 seq_num 1030) 2024-09-14T03:48:58.836 INFO:tasks.rados.rados.0.smithi110.stdout:3318: finishing write tid 1 to smithi11075562-1 2024-09-14T03:48:58.836 INFO:tasks.rados.rados.0.smithi110.stdout:3318: finishing write tid 2 to smithi11075562-1 2024-09-14T03:48:58.837 INFO:tasks.rados.rados.0.smithi110.stdout:3318: finishing write tid 3 to smithi11075562-1 2024-09-14T03:48:58.837 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2056 (ObjNum 1032 snap 328 seq_num 1032) dirty exists 2024-09-14T03:48:58.837 INFO:tasks.rados.rados.0.smithi110.stdout:3318: left oid 1 (ObjNum 1032 snap 328 seq_num 1032) 2024-09-14T03:48:58.838 INFO:tasks.rados.rados.0.smithi110.stdout:3316: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:58 smithi110 ceph-mon[86423]: pgmap v726: 121 pgs: 121 active+clean; 254 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-14T03:48:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:58 smithi110 ceph-mon[86423]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T03:48:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:48:58 smithi110 ceph-mon[86423]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T03:48:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:58 smithi110 ceph-mon[88262]: pgmap v726: 121 pgs: 121 active+clean; 254 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-14T03:48:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:58 smithi110 ceph-mon[88262]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T03:48:59.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:48:58 smithi110 ceph-mon[88262]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T03:48:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:58 smithi195 ceph-mon[82388]: pgmap v726: 121 pgs: 121 active+clean; 254 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-14T03:48:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:58 smithi195 ceph-mon[82388]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T03:48:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:48:58 smithi195 ceph-mon[82388]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T03:48:59.799 INFO:tasks.rados.rados.0.smithi110.stdout:3315: done (3 left) 2024-09-14T03:48:59.799 INFO:tasks.rados.rados.0.smithi110.stdout:3316: done (2 left) 2024-09-14T03:48:59.800 INFO:tasks.rados.rados.0.smithi110.stdout:3318: done (1 left) 2024-09-14T03:48:59.800 INFO:tasks.rados.rados.0.smithi110.stdout:3319: done (0 left) 2024-09-14T03:48:59.800 INFO:tasks.rados.rados.0.smithi110.stdout:3320: delete oid 26 current snap is 329 2024-09-14T03:48:59.804 INFO:tasks.rados.rados.0.smithi110.stdout:3320: done (0 left) 2024-09-14T03:48:59.804 INFO:tasks.rados.rados.0.smithi110.stdout:3321: copy_from oid 40 from oid 42 current snap is 329 2024-09-14T03:48:59.804 INFO:tasks.rados.rados.0.smithi110.stdout:3322: rmattr oid 28 current snap is 329 2024-09-14T03:48:59.804 INFO:tasks.rados.rados.0.smithi110.stdout:3323: delete oid 50 current snap is 329 2024-09-14T03:48:59.805 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 0 (ObjNum 969 snap 311 seq_num 969) dirty exists 2024-09-14T03:48:59.808 INFO:tasks.rados.rados.0.smithi110.stdout:3322: done (2 left) 2024-09-14T03:48:59.808 INFO:tasks.rados.rados.0.smithi110.stdout:3323: done (1 left) 2024-09-14T03:48:59.808 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 329 2024-09-14T03:48:59.808 INFO:tasks.rados.rados.0.smithi110.stdout:3324: seq_num 1035 ranges {950272=540672} 2024-09-14T03:48:59.817 INFO:tasks.rados.rados.0.smithi110.stdout:3324: writing smithi11075562-45 from 950272 to 1490944 tid 1 2024-09-14T03:48:59.817 INFO:tasks.rados.rados.0.smithi110.stdout:3325: read oid 14 snap -1 2024-09-14T03:48:59.817 INFO:tasks.rados.rados.0.smithi110.stdout:3325: expect (ObjNum 1003 snap 323 seq_num 1003) 2024-09-14T03:48:59.817 INFO:tasks.rados.rados.0.smithi110.stdout:3326: read oid 21 snap -1 2024-09-14T03:48:59.817 INFO:tasks.rados.rados.0.smithi110.stdout:3326: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:48:59.817 INFO:tasks.rados.rados.0.smithi110.stdout:3327: setattr oid 24 current snap is 329 2024-09-14T03:48:59.817 INFO:tasks.rados.rados.0.smithi110.stdout:3321: finishing copy_from racing read to smithi11075562-40 2024-09-14T03:48:59.818 INFO:tasks.rados.rados.0.smithi110.stdout:3321: finishing copy_from to smithi11075562-40 2024-09-14T03:48:59.818 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 2040 (ObjNum 1004 snap 323 seq_num 1004) dirty exists 2024-09-14T03:48:59.819 INFO:tasks.rados.rados.0.smithi110.stdout:3321: done (4 left) 2024-09-14T03:48:59.819 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 329 2024-09-14T03:48:59.819 INFO:tasks.rados.rados.0.smithi110.stdout:3328: seq_num 1036 ranges {2465792=557056} 2024-09-14T03:48:59.836 INFO:tasks.rados.rados.0.smithi110.stdout:3328: writing smithi11075562-1 from 2465792 to 3022848 tid 1 2024-09-14T03:48:59.836 INFO:tasks.rados.rados.0.smithi110.stdout:3324: finishing write tid 1 to smithi11075562-45 2024-09-14T03:48:59.836 INFO:tasks.rados.rados.0.smithi110.stdout:3329: read oid 35 snap -1 2024-09-14T03:48:59.837 INFO:tasks.rados.rados.0.smithi110.stdout:3329: expect deleted 2024-09-14T03:48:59.837 INFO:tasks.rados.rados.0.smithi110.stdout:3330: copy_from oid 44 from oid 28 current snap is 329 2024-09-14T03:48:59.837 INFO:tasks.rados.rados.0.smithi110.stdout:3324: finishing write tid 2 to smithi11075562-45 2024-09-14T03:48:59.837 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2009 (ObjNum 987 snap 318 seq_num 987) dirty exists 2024-09-14T03:48:59.837 INFO:tasks.rados.rados.0.smithi110.stdout:3324: finishing write tid 3 to smithi11075562-45 2024-09-14T03:48:59.837 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2337 (ObjNum 1035 snap 329 seq_num 1035) dirty exists 2024-09-14T03:48:59.837 INFO:tasks.rados.rados.0.smithi110.stdout:3324: left oid 45 (ObjNum 1035 snap 329 seq_num 1035) 2024-09-14T03:48:59.837 INFO:tasks.rados.rados.0.smithi110.stdout:3325: expect (ObjNum 1003 snap 323 seq_num 1003) 2024-09-14T03:48:59.900 INFO:tasks.rados.rados.0.smithi110.stdout:3328: finishing write tid 1 to smithi11075562-1 2024-09-14T03:48:59.900 INFO:tasks.rados.rados.0.smithi110.stdout:3328: finishing write tid 2 to smithi11075562-1 2024-09-14T03:48:59.901 INFO:tasks.rados.rados.0.smithi110.stdout:3328: finishing write tid 3 to smithi11075562-1 2024-09-14T03:48:59.901 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2059 (ObjNum 1036 snap 329 seq_num 1036) dirty exists 2024-09-14T03:48:59.901 INFO:tasks.rados.rados.0.smithi110.stdout:3328: left oid 1 (ObjNum 1036 snap 329 seq_num 1036) 2024-09-14T03:48:59.901 INFO:tasks.rados.rados.0.smithi110.stdout:3326: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:49:00.240 INFO:tasks.rados.rados.0.smithi110.stdout:3330: finishing copy_from racing read to smithi11075562-44 2024-09-14T03:49:00.240 INFO:tasks.rados.rados.0.smithi110.stdout:3330: finishing copy_from to smithi11075562-44 2024-09-14T03:49:00.240 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 1999 (ObjNum 969 snap 311 seq_num 969) dirty exists 2024-09-14T03:49:00.240 INFO:tasks.rados.rados.0.smithi110.stdout:3324: done (6 left) 2024-09-14T03:49:00.240 INFO:tasks.rados.rados.0.smithi110.stdout:3325: done (5 left) 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3326: done (4 left) 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3327: done (3 left) 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3328: done (2 left) 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3329: done (1 left) 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3330: done (0 left) 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3331: read oid 10 snap -1 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3331: expect (ObjNum 983 snap 315 seq_num 983) 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3332: read oid 22 snap 326 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3332: expect deleted 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3333: copy_from oid 16 from oid 50 current snap is 329 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3334: rollback oid 34 current snap is 329 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 321 2024-09-14T03:49:00.241 INFO:tasks.rados.rados.0.smithi110.stdout:3335: read oid 42 snap -1 2024-09-14T03:49:00.242 INFO:tasks.rados.rados.0.smithi110.stdout:3335: expect (ObjNum 1004 snap 323 seq_num 1004) 2024-09-14T03:49:00.242 INFO:tasks.rados.rados.0.smithi110.stdout:3336: read oid 21 snap -1 2024-09-14T03:49:00.242 INFO:tasks.rados.rados.0.smithi110.stdout:3336: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:49:00.242 INFO:tasks.rados.rados.0.smithi110.stdout:3337: snap_remove snap 328 2024-09-14T03:49:00.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:49:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:49:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:49:00.804 INFO:tasks.rados.rados.0.smithi110.stdout:3337: done (6 left) 2024-09-14T03:49:00.804 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 329 2024-09-14T03:49:00.804 INFO:tasks.rados.rados.0.smithi110.stdout:3338: seq_num 1037 ranges {0=573440} 2024-09-14T03:49:00.808 INFO:tasks.rados.rados.0.smithi110.stdout:3338: writing smithi11075562-18 from 0 to 573440 tid 1 2024-09-14T03:49:00.808 INFO:tasks.rados.rados.0.smithi110.stdout:3339: read oid 40 snap -1 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3339: expect (ObjNum 1004 snap 323 seq_num 1004) 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3340: rollback oid 6 current snap is 329 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 6 to 314 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3333: finishing copy_from to smithi11075562-16 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3333: got expected ENOENT (src dne) 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3333: finishing copy_from racing read to smithi11075562-16 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3332: done (8 left) 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3333: done (7 left) 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3341: snap_create 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3334: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3334: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:49:00.809 INFO:tasks.rados.rados.0.smithi110.stdout:3334: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:49:00.810 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 2147 (ObjNum 993 snap 320 seq_num 993) dirty exists 2024-09-14T03:49:00.810 INFO:tasks.rados.rados.0.smithi110.stdout:3331: expect (ObjNum 983 snap 315 seq_num 983) 2024-09-14T03:49:00.963 INFO:tasks.rados.rados.0.smithi110.stdout:3335: expect (ObjNum 1004 snap 323 seq_num 1004) 2024-09-14T03:49:01.060 INFO:tasks.rados.rados.0.smithi110.stdout:3336: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:49:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[88262]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T03:49:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:49:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:49:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:49:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:49:01.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[86423]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T03:49:01.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:49:01.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:49:01.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:49:01.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:00 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:49:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:00 smithi195 ceph-mon[82388]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T03:49:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:49:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:49:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:49:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:00 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:49:01.403 INFO:tasks.rados.rados.0.smithi110.stdout:3338: finishing write tid 1 to smithi11075562-18 2024-09-14T03:49:01.403 INFO:tasks.rados.rados.0.smithi110.stdout:3338: finishing write tid 2 to smithi11075562-18 2024-09-14T03:49:01.404 INFO:tasks.rados.rados.0.smithi110.stdout:3338: finishing write tid 3 to smithi11075562-18 2024-09-14T03:49:01.404 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 2137 (ObjNum 1037 snap 329 seq_num 1037) dirty exists 2024-09-14T03:49:01.404 INFO:tasks.rados.rados.0.smithi110.stdout:3338: left oid 18 (ObjNum 1037 snap 329 seq_num 1037) 2024-09-14T03:49:01.404 INFO:tasks.rados.rados.0.smithi110.stdout:3340: finishing rollback tid 0 to smithi11075562-6 2024-09-14T03:49:01.404 INFO:tasks.rados.rados.0.smithi110.stdout:3339: expect (ObjNum 1004 snap 323 seq_num 1004) 2024-09-14T03:49:01.499 INFO:tasks.rados.rados.0.smithi110.stdout:3340: finishing rollback tid 1 to smithi11075562-6 2024-09-14T03:49:01.499 INFO:tasks.rados.rados.0.smithi110.stdout:3340: finishing rollback tid 2 to smithi11075562-6 2024-09-14T03:49:01.499 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2140 (ObjNum 953 snap 307 seq_num 953) dirty exists 2024-09-14T03:49:01.808 INFO:tasks.rados.rados.0.smithi110.stdout:3331: done (7 left) 2024-09-14T03:49:01.808 INFO:tasks.rados.rados.0.smithi110.stdout:3334: done (6 left) 2024-09-14T03:49:01.808 INFO:tasks.rados.rados.0.smithi110.stdout:3335: done (5 left) 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:3336: done (4 left) 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:3338: done (3 left) 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:3339: done (2 left) 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:3340: done (1 left) 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:3341: done (0 left) 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:3342: read oid 11 snap -1 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:3342: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 330 2024-09-14T03:49:01.809 INFO:tasks.rados.rados.0.smithi110.stdout:3343: seq_num 1038 ranges {3227648=589824} 2024-09-14T03:49:01.829 INFO:tasks.rados.rados.0.smithi110.stdout:3343: writing smithi11075562-28 from 3227648 to 3817472 tid 1 2024-09-14T03:49:01.829 INFO:tasks.rados.rados.0.smithi110.stdout:3344: delete oid 49 current snap is 330 2024-09-14T03:49:01.829 INFO:tasks.rados.rados.0.smithi110.stdout:3342: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:49:01.900 INFO:tasks.rados.rados.0.smithi110.stdout:3343: finishing write tid 1 to smithi11075562-28 2024-09-14T03:49:01.900 INFO:tasks.rados.rados.0.smithi110.stdout:3343: finishing write tid 2 to smithi11075562-28 2024-09-14T03:49:01.900 INFO:tasks.rados.rados.0.smithi110.stdout:3343: finishing write tid 3 to smithi11075562-28 2024-09-14T03:49:01.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 2002 (ObjNum 1038 snap 330 seq_num 1038) dirty exists 2024-09-14T03:49:01.900 INFO:tasks.rados.rados.0.smithi110.stdout:3343: left oid 28 (ObjNum 1038 snap 330 seq_num 1038) 2024-09-14T03:49:01.901 INFO:tasks.rados.rados.0.smithi110.stdout:3342: done (2 left) 2024-09-14T03:49:01.901 INFO:tasks.rados.rados.0.smithi110.stdout:3343: done (1 left) 2024-09-14T03:49:01.901 INFO:tasks.rados.rados.0.smithi110.stdout:3344: done (0 left) 2024-09-14T03:49:01.901 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 330 2024-09-14T03:49:01.901 INFO:tasks.rados.rados.0.smithi110.stdout:3345: seq_num 1040 ranges {0=622592} 2024-09-14T03:49:01.903 INFO:tasks.rados.rados.0.smithi110.stdout:3345: writing smithi11075562-41 from 0 to 622592 tid 1 2024-09-14T03:49:01.903 INFO:tasks.rados.rados.0.smithi110.stdout:3346: setattr oid 16 current snap is 330 2024-09-14T03:49:01.905 INFO:tasks.rados.rados.0.smithi110.stdout:3347: rollback oid 45 current snap is 330 2024-09-14T03:49:01.905 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 321 2024-09-14T03:49:01.905 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 330 2024-09-14T03:49:01.906 INFO:tasks.rados.rados.0.smithi110.stdout:3348: seq_num 1041 ranges {3022848=638976} 2024-09-14T03:49:01.926 INFO:tasks.rados.rados.0.smithi110.stdout:3348: writing smithi11075562-1 from 3022848 to 3661824 tid 1 2024-09-14T03:49:01.926 INFO:tasks.rados.rados.0.smithi110.stdout:3349: read oid 34 snap -1 2024-09-14T03:49:01.926 INFO:tasks.rados.rados.0.smithi110.stdout:3349: expect (ObjNum 993 snap 320 seq_num 993) 2024-09-14T03:49:01.926 INFO:tasks.rados.rados.0.smithi110.stdout:3350: rollback oid 24 current snap is 330 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 316 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:3345: finishing write tid 1 to smithi11075562-41 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:3345: finishing write tid 2 to smithi11075562-41 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2141 (ObjNum 81 snap 0 seq_num 1) dirty exists 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:3345: finishing write tid 3 to smithi11075562-41 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 1917 (ObjNum 1040 snap 330 seq_num 1040) dirty exists 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:3345: left oid 41 (ObjNum 1040 snap 330 seq_num 1040) 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:3347: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:3345: done (5 left) 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:3346: done (4 left) 2024-09-14T03:49:01.927 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 330 2024-09-14T03:49:01.928 INFO:tasks.rados.rados.0.smithi110.stdout:3347: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:49:01.928 INFO:tasks.rados.rados.0.smithi110.stdout:3347: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:49:01.928 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2343 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-14T03:49:01.928 INFO:tasks.rados.rados.0.smithi110.stdout:3351: seq_num 1042 ranges {442368=655360} 2024-09-14T03:49:01.932 INFO:tasks.rados.rados.0.smithi110.stdout:3351: writing smithi11075562-48 from 442368 to 1097728 tid 1 2024-09-14T03:49:01.932 INFO:tasks.rados.rados.0.smithi110.stdout:3347: done (4 left) 2024-09-14T03:49:01.932 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 330 2024-09-14T03:49:01.933 INFO:tasks.rados.rados.0.smithi110.stdout:3348: finishing write tid 1 to smithi11075562-1 2024-09-14T03:49:01.933 INFO:tasks.rados.rados.0.smithi110.stdout:3352: seq_num 1043 ranges {1507328=671744} 2024-09-14T03:49:01.944 INFO:tasks.rados.rados.0.smithi110.stdout:3352: writing smithi11075562-19 from 1507328 to 2179072 tid 1 2024-09-14T03:49:01.945 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 330 2024-09-14T03:49:01.945 INFO:tasks.rados.rados.0.smithi110.stdout:3348: finishing write tid 2 to smithi11075562-1 2024-09-14T03:49:01.945 INFO:tasks.rados.rados.0.smithi110.stdout:3353: seq_num 1044 ranges {0=688128} 2024-09-14T03:49:01.945 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:49:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:49:01] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-14T03:49:01.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:01 smithi110 ceph-mon[88262]: pgmap v730: 121 pgs: 121 active+clean; 255 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 100 KiB/s wr, 7 op/s 2024-09-14T03:49:01.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:01 smithi110 ceph-mon[88262]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T03:49:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:01 smithi110 ceph-mon[86423]: pgmap v730: 121 pgs: 121 active+clean; 255 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 100 KiB/s wr, 7 op/s 2024-09-14T03:49:01.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:01 smithi110 ceph-mon[86423]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T03:49:01.948 INFO:tasks.rados.rados.0.smithi110.stdout:3353: writing smithi11075562-22 from 0 to 688128 tid 1 2024-09-14T03:49:01.949 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 330 2024-09-14T03:49:01.949 INFO:tasks.rados.rados.0.smithi110.stdout:3354: seq_num 1045 ranges {1736704=704512} 2024-09-14T03:49:01.962 INFO:tasks.rados.rados.0.smithi110.stdout:3354: writing smithi11075562-38 from 1736704 to 2441216 tid 1 2024-09-14T03:49:01.962 INFO:tasks.rados.rados.0.smithi110.stdout:3355: snap_remove snap 314 2024-09-14T03:49:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:01 smithi195 ceph-mon[82388]: pgmap v730: 121 pgs: 121 active+clean; 255 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 100 KiB/s wr, 7 op/s 2024-09-14T03:49:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:01 smithi195 ceph-mon[82388]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T03:49:02.815 INFO:tasks.rados.rados.0.smithi110.stdout:3355: done (7 left) 2024-09-14T03:49:02.815 INFO:tasks.rados.rados.0.smithi110.stdout:3356: read oid 3 snap -1 2024-09-14T03:49:02.815 INFO:tasks.rados.rados.0.smithi110.stdout:3356: expect (ObjNum 965 snap 311 seq_num 965) 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3357: delete oid 7 current snap is 330 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3348: finishing write tid 3 to smithi11075562-1 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2066 (ObjNum 1041 snap 330 seq_num 1041) dirty exists 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3348: left oid 1 (ObjNum 1041 snap 330 seq_num 1041) 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3350: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3350: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3350: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2012 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3351: finishing write tid 1 to smithi11075562-48 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3351: finishing write tid 2 to smithi11075562-48 2024-09-14T03:49:02.816 INFO:tasks.rados.rados.0.smithi110.stdout:3349: expect (ObjNum 993 snap 320 seq_num 993) 2024-09-14T03:49:02.918 INFO:tasks.rados.rados.0.smithi110.stdout:3351: finishing write tid 3 to smithi11075562-48 2024-09-14T03:49:02.918 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1998 (ObjNum 1042 snap 330 seq_num 1042) dirty exists 2024-09-14T03:49:02.918 INFO:tasks.rados.rados.0.smithi110.stdout:3351: left oid 48 (ObjNum 1042 snap 330 seq_num 1042) 2024-09-14T03:49:02.918 INFO:tasks.rados.rados.0.smithi110.stdout:3352: finishing write tid 1 to smithi11075562-19 2024-09-14T03:49:02.918 INFO:tasks.rados.rados.0.smithi110.stdout:3352: finishing write tid 2 to smithi11075562-19 2024-09-14T03:49:02.918 INFO:tasks.rados.rados.0.smithi110.stdout:3352: finishing write tid 3 to smithi11075562-19 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2143 (ObjNum 1043 snap 330 seq_num 1043) dirty exists 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3352: left oid 19 (ObjNum 1043 snap 330 seq_num 1043) 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3353: finishing write tid 1 to smithi11075562-22 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3348: done (8 left) 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3349: done (7 left) 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3350: done (6 left) 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3351: done (5 left) 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3352: done (4 left) 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3357: done (3 left) 2024-09-14T03:49:02.919 INFO:tasks.rados.rados.0.smithi110.stdout:3358: snap_remove snap 322 2024-09-14T03:49:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:02 smithi110 ceph-mon[88262]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T03:49:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:02 smithi110 ceph-mon[86423]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T03:49:03.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:02 smithi195 ceph-mon[82388]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T03:49:03.801 INFO:tasks.rados.rados.0.smithi110.stdout:3358: done (3 left) 2024-09-14T03:49:03.801 INFO:tasks.rados.rados.0.smithi110.stdout:3359: read oid 9 snap -1 2024-09-14T03:49:03.801 INFO:tasks.rados.rados.0.smithi110.stdout:3359: expect (ObjNum 994 snap 321 seq_num 994) 2024-09-14T03:49:03.801 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 330 2024-09-14T03:49:03.801 INFO:tasks.rados.rados.0.smithi110.stdout:3360: seq_num 1047 ranges {0=745472} 2024-09-14T03:49:03.806 INFO:tasks.rados.rados.0.smithi110.stdout:3360: writing smithi11075562-26 from 0 to 745472 tid 1 2024-09-14T03:49:03.806 INFO:tasks.rados.rados.0.smithi110.stdout:3361: read oid 17 snap 324 2024-09-14T03:49:03.806 INFO:tasks.rados.rados.0.smithi110.stdout:3361: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:49:03.806 INFO:tasks.rados.rados.0.smithi110.stdout:3362: read oid 47 snap -1 2024-09-14T03:49:03.807 INFO:tasks.rados.rados.0.smithi110.stdout:3362: expect deleted 2024-09-14T03:49:03.807 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 330 2024-09-14T03:49:03.807 INFO:tasks.rados.rados.0.smithi110.stdout:3363: seq_num 1048 ranges {0=761856} 2024-09-14T03:49:03.812 INFO:tasks.rados.rados.0.smithi110.stdout:3363: writing smithi11075562-32 from 0 to 761856 tid 1 2024-09-14T03:49:03.812 INFO:tasks.rados.rados.0.smithi110.stdout:3364: read oid 37 snap -1 2024-09-14T03:49:03.812 INFO:tasks.rados.rados.0.smithi110.stdout:3364: expect (ObjNum 1001 snap 323 seq_num 1001) 2024-09-14T03:49:03.812 INFO:tasks.rados.rados.0.smithi110.stdout:3365: copy_from oid 42 from oid 28 current snap is 330 2024-09-14T03:49:03.812 INFO:tasks.rados.rados.0.smithi110.stdout:3366: read oid 39 snap -1 2024-09-14T03:49:03.812 INFO:tasks.rados.rados.0.smithi110.stdout:3366: expect (ObjNum 970 snap 312 seq_num 970) 2024-09-14T03:49:03.813 INFO:tasks.rados.rados.0.smithi110.stdout:3367: snap_remove snap 321 2024-09-14T03:49:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:03 smithi110 ceph-mon[88262]: pgmap v733: 121 pgs: 121 active+clean; 268 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-14T03:49:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:03 smithi110 ceph-mon[88262]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T03:49:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:03 smithi110 ceph-mon[88262]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T03:49:04.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:03 smithi110 ceph-mon[86423]: pgmap v733: 121 pgs: 121 active+clean; 268 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-14T03:49:04.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:03 smithi110 ceph-mon[86423]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T03:49:04.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:03 smithi110 ceph-mon[86423]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T03:49:04.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:03 smithi195 ceph-mon[82388]: pgmap v733: 121 pgs: 121 active+clean; 268 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-14T03:49:04.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:03 smithi195 ceph-mon[82388]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T03:49:04.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:03 smithi195 ceph-mon[82388]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:3367: done (11 left) 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 330 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:3353: finishing write tid 2 to smithi11075562-22 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:3353: finishing write tid 3 to smithi11075562-22 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2041 (ObjNum 1044 snap 330 seq_num 1044) dirty exists 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:3353: left oid 22 (ObjNum 1044 snap 330 seq_num 1044) 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:3354: finishing write tid 1 to smithi11075562-38 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:3354: finishing write tid 2 to smithi11075562-38 2024-09-14T03:49:04.818 INFO:tasks.rados.rados.0.smithi110.stdout:3354: finishing write tid 3 to smithi11075562-38 2024-09-14T03:49:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1632 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:49:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:3354: left oid 38 (ObjNum 1045 snap 330 seq_num 1045) 2024-09-14T03:49:04.819 INFO:tasks.rados.rados.0.smithi110.stdout:3356: expect (ObjNum 965 snap 311 seq_num 965) 2024-09-14T03:49:04.875 INFO:tasks.rados.rados.0.smithi110.stdout:3363: finishing write tid 1 to smithi11075562-32 2024-09-14T03:49:04.875 INFO:tasks.rados.rados.0.smithi110.stdout:3363: finishing write tid 2 to smithi11075562-32 2024-09-14T03:49:04.875 INFO:tasks.rados.rados.0.smithi110.stdout:3363: finishing write tid 3 to smithi11075562-32 2024-09-14T03:49:04.875 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2043 (ObjNum 1048 snap 330 seq_num 1048) dirty exists 2024-09-14T03:49:04.875 INFO:tasks.rados.rados.0.smithi110.stdout:3363: left oid 32 (ObjNum 1048 snap 330 seq_num 1048) 2024-09-14T03:49:04.876 INFO:tasks.rados.rados.0.smithi110.stdout:3360: finishing write tid 1 to smithi11075562-26 2024-09-14T03:49:04.876 INFO:tasks.rados.rados.0.smithi110.stdout:3360: finishing write tid 2 to smithi11075562-26 2024-09-14T03:49:04.876 INFO:tasks.rados.rados.0.smithi110.stdout:3368: seq_num 1049 ranges {3858432=778240} 2024-09-14T03:49:04.900 INFO:tasks.rados.rados.0.smithi110.stdout:3368: writing smithi11075562-10 from 3858432 to 4636672 tid 1 2024-09-14T03:49:04.900 INFO:tasks.rados.rados.0.smithi110.stdout:3353: done (11 left) 2024-09-14T03:49:04.900 INFO:tasks.rados.rados.0.smithi110.stdout:3354: done (10 left) 2024-09-14T03:49:04.900 INFO:tasks.rados.rados.0.smithi110.stdout:3356: done (9 left) 2024-09-14T03:49:04.900 INFO:tasks.rados.rados.0.smithi110.stdout:3362: done (8 left) 2024-09-14T03:49:04.900 INFO:tasks.rados.rados.0.smithi110.stdout:3363: done (7 left) 2024-09-14T03:49:04.900 INFO:tasks.rados.rados.0.smithi110.stdout:3369: snap_create 2024-09-14T03:49:04.900 INFO:tasks.rados.rados.0.smithi110.stdout:3359: expect (ObjNum 994 snap 321 seq_num 994) 2024-09-14T03:49:04.962 INFO:tasks.rados.rados.0.smithi110.stdout:3360: finishing write tid 3 to smithi11075562-26 2024-09-14T03:49:04.962 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1850 (ObjNum 1047 snap 330 seq_num 1047) dirty exists 2024-09-14T03:49:04.962 INFO:tasks.rados.rados.0.smithi110.stdout:3360: left oid 26 (ObjNum 1047 snap 330 seq_num 1047) 2024-09-14T03:49:04.962 INFO:tasks.rados.rados.0.smithi110.stdout:3364: expect (ObjNum 1001 snap 323 seq_num 1001) 2024-09-14T03:49:05.024 INFO:tasks.rados.rados.0.smithi110.stdout:3361: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:49:05.361 INFO:tasks.rados.rados.0.smithi110.stdout:3365: finishing copy_from racing read to smithi11075562-42 2024-09-14T03:49:05.361 INFO:tasks.rados.rados.0.smithi110.stdout:3365: finishing copy_from to smithi11075562-42 2024-09-14T03:49:05.361 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2002 (ObjNum 1038 snap 330 seq_num 1038) dirty exists 2024-09-14T03:49:05.361 INFO:tasks.rados.rados.0.smithi110.stdout:3366: expect (ObjNum 970 snap 312 seq_num 970) 2024-09-14T03:49:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:05 smithi110 ceph-mon[88262]: pgmap v736: 121 pgs: 121 active+clean; 270 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:49:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:05 smithi110 ceph-mon[88262]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T03:49:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:05 smithi110 ceph-mon[86423]: pgmap v736: 121 pgs: 121 active+clean; 270 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:49:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:05 smithi110 ceph-mon[86423]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T03:49:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:05 smithi195 ceph-mon[82388]: pgmap v736: 121 pgs: 121 active+clean; 270 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 20 op/s 2024-09-14T03:49:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:05 smithi195 ceph-mon[82388]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T03:49:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:06 smithi110 ceph-mon[88262]: pgmap v738: 121 pgs: 121 active+clean; 258 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T03:49:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:06 smithi110 ceph-mon[88262]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T03:49:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:06 smithi110 ceph-mon[86423]: pgmap v738: 121 pgs: 121 active+clean; 258 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T03:49:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:06 smithi110 ceph-mon[86423]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T03:49:07.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:06 smithi195 ceph-mon[82388]: pgmap v738: 121 pgs: 121 active+clean; 258 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T03:49:07.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:06 smithi195 ceph-mon[82388]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T03:49:07.336 INFO:tasks.rados.rados.0.smithi110.stdout:3368: finishing write tid 1 to smithi11075562-10 2024-09-14T03:49:07.336 INFO:tasks.rados.rados.0.smithi110.stdout:3368: finishing write tid 2 to smithi11075562-10 2024-09-14T03:49:07.336 INFO:tasks.rados.rados.0.smithi110.stdout:3368: finishing write tid 3 to smithi11075562-10 2024-09-14T03:49:07.336 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2014 (ObjNum 1049 snap 330 seq_num 1049) dirty exists 2024-09-14T03:49:07.336 INFO:tasks.rados.rados.0.smithi110.stdout:3368: left oid 10 (ObjNum 1049 snap 330 seq_num 1049) 2024-09-14T03:49:07.336 INFO:tasks.rados.rados.0.smithi110.stdout:3359: done (7 left) 2024-09-14T03:49:07.336 INFO:tasks.rados.rados.0.smithi110.stdout:3360: done (6 left) 2024-09-14T03:49:07.336 INFO:tasks.rados.rados.0.smithi110.stdout:3361: done (5 left) 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3364: done (4 left) 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3365: done (3 left) 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3366: done (2 left) 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3368: done (1 left) 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3369: done (0 left) 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3370: rmattr oid 43 current snap is 331 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3371: read oid 38 snap 326 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3371: expect (ObjNum 963 snap 310 seq_num 963) 2024-09-14T03:49:07.337 INFO:tasks.rados.rados.0.smithi110.stdout:3372: delete oid 13 current snap is 331 2024-09-14T03:49:07.340 INFO:tasks.rados.rados.0.smithi110.stdout:3372: done (2 left) 2024-09-14T03:49:07.340 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 331 2024-09-14T03:49:07.340 INFO:tasks.rados.rados.0.smithi110.stdout:3373: seq_num 1051 ranges {0=409600,409600=401408} 2024-09-14T03:49:07.342 INFO:tasks.rados.rados.0.smithi110.stdout:3373: writing smithi11075562-35 from 0 to 409600 tid 1 2024-09-14T03:49:07.345 INFO:tasks.rados.rados.0.smithi110.stdout:3373: writing smithi11075562-35 from 409600 to 811008 tid 2 2024-09-14T03:49:07.345 INFO:tasks.rados.rados.0.smithi110.stdout:3374: copy_from oid 40 from oid 33 current snap is 331 2024-09-14T03:49:07.345 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1918 (ObjNum 828 snap 274 seq_num 828) dirty exists 2024-09-14T03:49:07.345 INFO:tasks.rados.rados.0.smithi110.stdout:3371: expect (ObjNum 963 snap 310 seq_num 963) 2024-09-14T03:49:07.390 INFO:tasks.rados.rados.0.smithi110.stdout:3373: finishing write tid 1 to smithi11075562-35 2024-09-14T03:49:07.390 INFO:tasks.rados.rados.0.smithi110.stdout:3373: finishing write tid 2 to smithi11075562-35 2024-09-14T03:49:07.390 INFO:tasks.rados.rados.0.smithi110.stdout:3373: finishing write tid 3 to smithi11075562-35 2024-09-14T03:49:07.390 INFO:tasks.rados.rados.0.smithi110.stdout:3374: finishing copy_from racing read to smithi11075562-40 2024-09-14T03:49:07.390 INFO:tasks.rados.rados.0.smithi110.stdout:3373: finishing write tid 4 to smithi11075562-35 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 2097 (ObjNum 1051 snap 331 seq_num 1051) dirty exists 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:3373: left oid 35 (ObjNum 1051 snap 331 seq_num 1051) 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:3374: finishing copy_from to smithi11075562-40 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 2134 (ObjNum 753 snap 0 seq_num 64678784) dirty exists 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:3370: done (3 left) 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:3371: done (2 left) 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:3373: done (1 left) 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:3374: done (0 left) 2024-09-14T03:49:07.391 INFO:tasks.rados.rados.0.smithi110.stdout:3375: delete oid 41 current snap is 331 2024-09-14T03:49:07.392 INFO:tasks.rados.rados.0.smithi110.stdout:3375: done (0 left) 2024-09-14T03:49:07.392 INFO:tasks.rados.rados.0.smithi110.stdout:3376: read oid 12 snap 329 2024-09-14T03:49:07.392 INFO:tasks.rados.rados.0.smithi110.stdout:3376: expect deleted 2024-09-14T03:49:07.392 INFO:tasks.rados.rados.0.smithi110.stdout:3377: snap_remove snap 330 2024-09-14T03:49:07.842 INFO:tasks.rados.rados.0.smithi110.stdout:3377: done (1 left) 2024-09-14T03:49:07.842 INFO:tasks.rados.rados.0.smithi110.stdout:3378: snap_create 2024-09-14T03:49:08.802 INFO:tasks.rados.rados.0.smithi110.stdout:3376: done (1 left) 2024-09-14T03:49:08.802 INFO:tasks.rados.rados.0.smithi110.stdout:3378: done (0 left) 2024-09-14T03:49:08.803 INFO:tasks.rados.rados.0.smithi110.stdout:3379: read oid 37 snap -1 2024-09-14T03:49:08.803 INFO:tasks.rados.rados.0.smithi110.stdout:3379: expect (ObjNum 1001 snap 323 seq_num 1001) 2024-09-14T03:49:08.803 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 332 2024-09-14T03:49:08.803 INFO:tasks.rados.rados.0.smithi110.stdout:3380: seq_num 1053 ranges {688128=442368,1130496=401408} 2024-09-14T03:49:08.808 INFO:tasks.rados.rados.0.smithi110.stdout:3380: writing smithi11075562-22 from 688128 to 1130496 tid 1 2024-09-14T03:49:08.811 INFO:tasks.rados.rados.0.smithi110.stdout:3380: writing smithi11075562-22 from 1130496 to 1531904 tid 2 2024-09-14T03:49:08.811 INFO:tasks.rados.rados.0.smithi110.stdout:3381: copy_from oid 47 from oid 50 current snap is 332 2024-09-14T03:49:08.811 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 48 current snap is 332 2024-09-14T03:49:08.811 INFO:tasks.rados.rados.0.smithi110.stdout:3382: seq_num 1054 ranges {1097728=458752,1556480=401408} 2024-09-14T03:49:08.819 INFO:tasks.rados.rados.0.smithi110.stdout:3382: writing smithi11075562-48 from 1097728 to 1556480 tid 1 2024-09-14T03:49:08.822 INFO:tasks.rados.rados.0.smithi110.stdout:3382: writing smithi11075562-48 from 1556480 to 1957888 tid 2 2024-09-14T03:49:08.822 INFO:tasks.rados.rados.0.smithi110.stdout:3383: setattr oid 24 current snap is 332 2024-09-14T03:49:08.822 INFO:tasks.rados.rados.0.smithi110.stdout:3379: expect (ObjNum 1001 snap 323 seq_num 1001) 2024-09-14T03:49:08.841 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:08 smithi110 ceph-mon[88262]: pgmap v740: 121 pgs: 121 active+clean; 247 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 26 op/s 2024-09-14T03:49:08.842 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:08 smithi110 ceph-mon[86423]: pgmap v740: 121 pgs: 121 active+clean; 247 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 26 op/s 2024-09-14T03:49:08.842 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:08 smithi110 ceph-mon[86423]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T03:49:08.842 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:08 smithi110 ceph-mon[86423]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T03:49:08.889 INFO:tasks.rados.rados.0.smithi110.stdout:3381: finishing copy_from to smithi11075562-47 2024-09-14T03:49:08.889 INFO:tasks.rados.rados.0.smithi110.stdout:3381: got expected ENOENT (src dne) 2024-09-14T03:49:08.889 INFO:tasks.rados.rados.0.smithi110.stdout:3381: finishing copy_from racing read to smithi11075562-47 2024-09-14T03:49:08.889 INFO:tasks.rados.rados.0.smithi110.stdout:3380: finishing write tid 1 to smithi11075562-22 2024-09-14T03:49:08.889 INFO:tasks.rados.rados.0.smithi110.stdout:3380: finishing write tid 2 to smithi11075562-22 2024-09-14T03:49:08.889 INFO:tasks.rados.rados.0.smithi110.stdout:3380: finishing write tid 3 to smithi11075562-22 2024-09-14T03:49:08.889 INFO:tasks.rados.rados.0.smithi110.stdout:3380: finishing write tid 4 to smithi11075562-22 2024-09-14T03:49:08.889 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2046 (ObjNum 1053 snap 332 seq_num 1053) dirty exists 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3380: left oid 22 (ObjNum 1053 snap 332 seq_num 1053) 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3382: finishing write tid 1 to smithi11075562-48 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3382: finishing write tid 2 to smithi11075562-48 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3382: finishing write tid 3 to smithi11075562-48 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3382: finishing write tid 4 to smithi11075562-48 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 2001 (ObjNum 1054 snap 332 seq_num 1054) dirty exists 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3382: left oid 48 (ObjNum 1054 snap 332 seq_num 1054) 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3379: done (4 left) 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3380: done (3 left) 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3381: done (2 left) 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3382: done (1 left) 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3384: read oid 37 snap -1 2024-09-14T03:49:08.890 INFO:tasks.rados.rados.0.smithi110.stdout:3384: expect (ObjNum 1001 snap 323 seq_num 1001) 2024-09-14T03:49:08.891 INFO:tasks.rados.rados.0.smithi110.stdout:3385: rollback oid 27 current snap is 332 2024-09-14T03:49:08.891 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 329 2024-09-14T03:49:08.891 INFO:tasks.rados.rados.0.smithi110.stdout:3386: setattr oid 47 current snap is 332 2024-09-14T03:49:08.891 INFO:tasks.rados.rados.0.smithi110.stdout:3385: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:49:08.891 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 0 (ObjNum 33 snap 0 seq_num 66385840) dirty dne 2024-09-14T03:49:08.891 INFO:tasks.rados.rados.0.smithi110.stdout:3385: done (3 left) 2024-09-14T03:49:08.891 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 18 current snap is 332 2024-09-14T03:49:08.891 INFO:tasks.rados.rados.0.smithi110.stdout:3387: seq_num 1055 ranges {573440=475136,1048576=401408} 2024-09-14T03:49:08.895 INFO:tasks.rados.rados.0.smithi110.stdout:3387: writing smithi11075562-18 from 573440 to 1048576 tid 1 2024-09-14T03:49:08.898 INFO:tasks.rados.rados.0.smithi110.stdout:3387: writing smithi11075562-18 from 1048576 to 1449984 tid 2 2024-09-14T03:49:08.898 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 11 current snap is 332 2024-09-14T03:49:08.898 INFO:tasks.rados.rados.0.smithi110.stdout:3388: seq_num 1056 ranges {2080768=491520,2572288=401408} 2024-09-14T03:49:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3388: writing smithi11075562-11 from 2080768 to 2572288 tid 1 2024-09-14T03:49:08.914 INFO:tasks.rados.rados.0.smithi110.stdout:3388: writing smithi11075562-11 from 2572288 to 2973696 tid 2 2024-09-14T03:49:08.914 INFO:tasks.rados.rados.0.smithi110.stdout:3389: delete oid 49 current snap is 332 2024-09-14T03:49:08.914 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2015 (ObjNum 809 snap 269 seq_num 809) dirty exists 2024-09-14T03:49:08.914 INFO:tasks.rados.rados.0.smithi110.stdout:3384: expect (ObjNum 1001 snap 323 seq_num 1001) 2024-09-14T03:49:08.977 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 2376 (ObjNum 33 snap 0 seq_num 65312032) dirty exists 2024-09-14T03:49:08.977 INFO:tasks.rados.rados.0.smithi110.stdout:3387: finishing write tid 1 to smithi11075562-18 2024-09-14T03:49:08.977 INFO:tasks.rados.rados.0.smithi110.stdout:3387: finishing write tid 2 to smithi11075562-18 2024-09-14T03:49:08.977 INFO:tasks.rados.rados.0.smithi110.stdout:3387: finishing write tid 3 to smithi11075562-18 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3387: finishing write tid 4 to smithi11075562-18 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 2146 (ObjNum 1055 snap 332 seq_num 1055) dirty exists 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3387: left oid 18 (ObjNum 1055 snap 332 seq_num 1055) 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3388: finishing write tid 1 to smithi11075562-11 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3388: finishing write tid 2 to smithi11075562-11 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3383: done (5 left) 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3384: done (4 left) 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3386: done (3 left) 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3387: done (2 left) 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3389: done (1 left) 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 332 2024-09-14T03:49:08.978 INFO:tasks.rados.rados.0.smithi110.stdout:3388: finishing write tid 3 to smithi11075562-11 2024-09-14T03:49:08.979 INFO:tasks.rados.rados.0.smithi110.stdout:3388: finishing write tid 4 to smithi11075562-11 2024-09-14T03:49:08.979 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 2004 (ObjNum 1056 snap 332 seq_num 1056) dirty exists 2024-09-14T03:49:08.979 INFO:tasks.rados.rados.0.smithi110.stdout:3388: left oid 11 (ObjNum 1056 snap 332 seq_num 1056) 2024-09-14T03:49:08.979 INFO:tasks.rados.rados.0.smithi110.stdout:3390: seq_num 1058 ranges {4636672=524288,5160960=401408} 2024-09-14T03:49:09.003 INFO:tasks.rados.rados.0.smithi110.stdout:3390: writing smithi11075562-10 from 4636672 to 5160960 tid 1 2024-09-14T03:49:09.006 INFO:tasks.rados.rados.0.smithi110.stdout:3390: writing smithi11075562-10 from 5160960 to 5562368 tid 2 2024-09-14T03:49:09.006 INFO:tasks.rados.rados.0.smithi110.stdout:3388: done (1 left) 2024-09-14T03:49:09.006 INFO:tasks.rados.rados.0.smithi110.stdout:3391: snap_create 2024-09-14T03:49:09.008 INFO:tasks.rados.rados.0.smithi110.stdout:3390: finishing write tid 1 to smithi11075562-10 2024-09-14T03:49:09.008 INFO:tasks.rados.rados.0.smithi110.stdout:3390: finishing write tid 2 to smithi11075562-10 2024-09-14T03:49:09.008 INFO:tasks.rados.rados.0.smithi110.stdout:3390: finishing write tid 3 to smithi11075562-10 2024-09-14T03:49:09.009 INFO:tasks.rados.rados.0.smithi110.stdout:3390: finishing write tid 4 to smithi11075562-10 2024-09-14T03:49:09.009 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2018 (ObjNum 1058 snap 332 seq_num 1058) dirty exists 2024-09-14T03:49:09.009 INFO:tasks.rados.rados.0.smithi110.stdout:3390: left oid 10 (ObjNum 1058 snap 332 seq_num 1058) 2024-09-14T03:49:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:08 smithi110 ceph-mon[88262]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T03:49:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:08 smithi110 ceph-mon[88262]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T03:49:09.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:08 smithi195 ceph-mon[82388]: pgmap v740: 121 pgs: 121 active+clean; 247 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 26 op/s 2024-09-14T03:49:09.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:08 smithi195 ceph-mon[82388]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T03:49:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:08 smithi195 ceph-mon[82388]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T03:49:09.849 INFO:tasks.rados.rados.0.smithi110.stdout:3390: done (1 left) 2024-09-14T03:49:09.849 INFO:tasks.rados.rados.0.smithi110.stdout:3391: done (0 left) 2024-09-14T03:49:09.849 INFO:tasks.rados.rados.0.smithi110.stdout:3392: read oid 2 snap 316 2024-09-14T03:49:09.849 INFO:tasks.rados.rados.0.smithi110.stdout:3392: expect (ObjNum 935 snap 302 seq_num 935) 2024-09-14T03:49:09.849 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 333 2024-09-14T03:49:09.849 INFO:tasks.rados.rados.0.smithi110.stdout:3393: seq_num 1059 ranges {0=540672,540672=401408} 2024-09-14T03:49:09.852 INFO:tasks.rados.rados.0.smithi110.stdout:3393: writing smithi11075562-12 from 0 to 540672 tid 1 2024-09-14T03:49:09.857 INFO:tasks.rados.rados.0.smithi110.stdout:3393: writing smithi11075562-12 from 540672 to 942080 tid 2 2024-09-14T03:49:09.857 INFO:tasks.rados.rados.0.smithi110.stdout:3394: read oid 18 snap -1 2024-09-14T03:49:09.857 INFO:tasks.rados.rados.0.smithi110.stdout:3394: expect (ObjNum 1055 snap 332 seq_num 1055) 2024-09-14T03:49:09.857 INFO:tasks.rados.rados.0.smithi110.stdout:3395: read oid 21 snap -1 2024-09-14T03:49:09.857 INFO:tasks.rados.rados.0.smithi110.stdout:3395: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:49:09.857 INFO:tasks.rados.rados.0.smithi110.stdout:3396: read oid 16 snap 327 2024-09-14T03:49:09.857 INFO:tasks.rados.rados.0.smithi110.stdout:3396: expect deleted 2024-09-14T03:49:09.857 INFO:tasks.rados.rados.0.smithi110.stdout:3397: snap_remove snap 323 2024-09-14T03:49:10.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:49:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:49:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:49:10.854 INFO:tasks.rados.rados.0.smithi110.stdout:3397: done (5 left) 2024-09-14T03:49:10.854 INFO:tasks.rados.rados.0.smithi110.stdout:3398: snap_create 2024-09-14T03:49:10.854 INFO:tasks.rados.rados.0.smithi110.stdout:3392: expect (ObjNum 935 snap 302 seq_num 935) 2024-09-14T03:49:10.907 INFO:tasks.rados.rados.0.smithi110.stdout:3393: finishing write tid 1 to smithi11075562-12 2024-09-14T03:49:10.907 INFO:tasks.rados.rados.0.smithi110.stdout:3393: finishing write tid 2 to smithi11075562-12 2024-09-14T03:49:10.908 INFO:tasks.rados.rados.0.smithi110.stdout:3393: finishing write tid 3 to smithi11075562-12 2024-09-14T03:49:10.908 INFO:tasks.rados.rados.0.smithi110.stdout:3393: finishing write tid 4 to smithi11075562-12 2024-09-14T03:49:10.908 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2379 (ObjNum 1059 snap 333 seq_num 1059) dirty exists 2024-09-14T03:49:10.908 INFO:tasks.rados.rados.0.smithi110.stdout:3393: left oid 12 (ObjNum 1059 snap 333 seq_num 1059) 2024-09-14T03:49:10.908 INFO:tasks.rados.rados.0.smithi110.stdout:3394: expect (ObjNum 1055 snap 332 seq_num 1055) 2024-09-14T03:49:10.956 INFO:tasks.rados.rados.0.smithi110.stdout:3395: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-14T03:49:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:10 smithi110 ceph-mon[88262]: pgmap v743: 121 pgs: 121 active+clean; 242 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T03:49:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:10 smithi110 ceph-mon[88262]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T03:49:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:10 smithi110 ceph-mon[86423]: pgmap v743: 121 pgs: 121 active+clean; 242 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T03:49:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:10 smithi110 ceph-mon[86423]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T03:49:11.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:10 smithi195 ceph-mon[82388]: pgmap v743: 121 pgs: 121 active+clean; 242 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T03:49:11.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:10 smithi195 ceph-mon[82388]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T03:49:11.855 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:49:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:49:11] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-14T03:49:11.859 INFO:tasks.rados.rados.0.smithi110.stdout:3392: done (5 left) 2024-09-14T03:49:11.860 INFO:tasks.rados.rados.0.smithi110.stdout:3393: done (4 left) 2024-09-14T03:49:11.860 INFO:tasks.rados.rados.0.smithi110.stdout:3394: done (3 left) 2024-09-14T03:49:11.860 INFO:tasks.rados.rados.0.smithi110.stdout:3395: done (2 left) 2024-09-14T03:49:11.860 INFO:tasks.rados.rados.0.smithi110.stdout:3396: done (1 left) 2024-09-14T03:49:11.860 INFO:tasks.rados.rados.0.smithi110.stdout:3398: done (0 left) 2024-09-14T03:49:11.860 INFO:tasks.rados.rados.0.smithi110.stdout:3399: read oid 48 snap 316 2024-09-14T03:49:11.860 INFO:tasks.rados.rados.0.smithi110.stdout:3399: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:49:11.860 INFO:tasks.rados.rados.0.smithi110.stdout:3400: snap_remove snap 333 2024-09-14T03:49:12.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:11 smithi110 ceph-mon[88262]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T03:49:12.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:11 smithi110 ceph-mon[86423]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T03:49:12.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:11 smithi195 ceph-mon[82388]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T03:49:12.862 INFO:tasks.rados.rados.0.smithi110.stdout:3400: done (1 left) 2024-09-14T03:49:12.862 INFO:tasks.rados.rados.0.smithi110.stdout:3401: copy_from oid 37 from oid 12 current snap is 334 2024-09-14T03:49:12.862 INFO:tasks.rados.rados.0.smithi110.stdout:3399: expect (ObjNum 809 snap 269 seq_num 809) 2024-09-14T03:49:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:12 smithi110 ceph-mon[88262]: pgmap v746: 121 pgs: 121 active+clean; 244 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-09-14T03:49:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:12 smithi110 ceph-mon[88262]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T03:49:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:12 smithi110 ceph-mon[86423]: pgmap v746: 121 pgs: 121 active+clean; 244 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-09-14T03:49:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:12 smithi110 ceph-mon[86423]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T03:49:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:13.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:12 smithi195 ceph-mon[82388]: pgmap v746: 121 pgs: 121 active+clean; 244 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-09-14T03:49:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:12 smithi195 ceph-mon[82388]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T03:49:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:13.421 INFO:tasks.rados.rados.0.smithi110.stdout:3401: finishing copy_from racing read to smithi11075562-37 2024-09-14T03:49:13.421 INFO:tasks.rados.rados.0.smithi110.stdout:3401: finishing copy_from to smithi11075562-37 2024-09-14T03:49:13.422 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 2379 (ObjNum 1059 snap 333 seq_num 1059) dirty exists 2024-09-14T03:49:13.422 INFO:tasks.rados.rados.0.smithi110.stdout:3399: done (1 left) 2024-09-14T03:49:13.422 INFO:tasks.rados.rados.0.smithi110.stdout:3401: done (0 left) 2024-09-14T03:49:13.422 INFO:tasks.rados.rados.0.smithi110.stdout:3402: snap_create 2024-09-14T03:49:13.802 INFO:tasks.rados.rados.0.smithi110.stdout:3402: done (0 left) 2024-09-14T03:49:13.803 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 335 2024-09-14T03:49:13.803 INFO:tasks.rados.rados.0.smithi110.stdout:3403: seq_num 1060 ranges {0=557056,557056=401408} 2024-09-14T03:49:13.805 INFO:tasks.rados.rados.0.smithi110.stdout:3403: writing smithi11075562-7 from 0 to 557056 tid 1 2024-09-14T03:49:13.808 INFO:tasks.rados.rados.0.smithi110.stdout:3403: writing smithi11075562-7 from 557056 to 958464 tid 2 2024-09-14T03:49:13.809 INFO:tasks.rados.rados.0.smithi110.stdout:3404: copy_from oid 48 from oid 38 current snap is 335 2024-09-14T03:49:13.809 INFO:tasks.rados.rados.0.smithi110.stdout:3405: copy_from oid 29 from oid 8 current snap is 335 2024-09-14T03:49:13.809 INFO:tasks.rados.rados.0.smithi110.stdout:3406: read oid 14 snap 316 2024-09-14T03:49:13.809 INFO:tasks.rados.rados.0.smithi110.stdout:3406: expect deleted 2024-09-14T03:49:13.809 INFO:tasks.rados.rados.0.smithi110.stdout:3407: snap_remove snap 334 2024-09-14T03:49:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:13 smithi110 ceph-mon[88262]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T03:49:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:13 smithi110 ceph-mon[88262]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T03:49:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:13 smithi110 ceph-mon[86423]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T03:49:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:13 smithi110 ceph-mon[86423]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T03:49:14.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:13 smithi195 ceph-mon[82388]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T03:49:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:13 smithi195 ceph-mon[82388]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T03:49:14.809 INFO:tasks.rados.rados.0.smithi110.stdout:3407: done (4 left) 2024-09-14T03:49:14.809 INFO:tasks.rados.rados.0.smithi110.stdout:3408: copy_from oid 43 from oid 18 current snap is 335 2024-09-14T03:49:14.809 INFO:tasks.rados.rados.0.smithi110.stdout:3403: finishing write tid 1 to smithi11075562-7 2024-09-14T03:49:14.809 INFO:tasks.rados.rados.0.smithi110.stdout:3403: finishing write tid 2 to smithi11075562-7 2024-09-14T03:49:14.809 INFO:tasks.rados.rados.0.smithi110.stdout:3403: finishing write tid 3 to smithi11075562-7 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:3405: finishing copy_from to smithi11075562-29 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:3405: got expected ENOENT (src dne) 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:3405: finishing copy_from racing read to smithi11075562-29 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:3403: finishing write tid 4 to smithi11075562-7 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1636 (ObjNum 1060 snap 335 seq_num 1060) dirty exists 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:3403: left oid 7 (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:3404: finishing copy_from racing read to smithi11075562-48 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:3404: finishing copy_from to smithi11075562-48 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 48 v 1632 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:49:14.810 INFO:tasks.rados.rados.0.smithi110.stdout:3403: done (4 left) 2024-09-14T03:49:14.811 INFO:tasks.rados.rados.0.smithi110.stdout:3404: done (3 left) 2024-09-14T03:49:14.811 INFO:tasks.rados.rados.0.smithi110.stdout:3405: done (2 left) 2024-09-14T03:49:14.811 INFO:tasks.rados.rados.0.smithi110.stdout:3406: done (1 left) 2024-09-14T03:49:14.811 INFO:tasks.rados.rados.0.smithi110.stdout:3409: read oid 12 snap -1 2024-09-14T03:49:14.811 INFO:tasks.rados.rados.0.smithi110.stdout:3409: expect (ObjNum 1059 snap 333 seq_num 1059) 2024-09-14T03:49:14.811 INFO:tasks.rados.rados.0.smithi110.stdout:3410: copy_from oid 44 from oid 4 current snap is 335 2024-09-14T03:49:14.811 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 335 2024-09-14T03:49:14.811 INFO:tasks.rados.rados.0.smithi110.stdout:3411: seq_num 1061 ranges {17260544=573440,17833984=401408} 2024-09-14T03:49:14.906 INFO:tasks.rados.rados.0.smithi110.stdout:3411: writing smithi11075562-39 from 17260544 to 17833984 tid 1 2024-09-14T03:49:14.909 INFO:tasks.rados.rados.0.smithi110.stdout:3411: writing smithi11075562-39 from 17833984 to 18235392 tid 2 2024-09-14T03:49:14.909 INFO:tasks.rados.rados.0.smithi110.stdout:3412: rollback oid 26 current snap is 335 2024-09-14T03:49:14.910 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 327 2024-09-14T03:49:14.910 INFO:tasks.rados.rados.0.smithi110.stdout:3409: expect (ObjNum 1059 snap 333 seq_num 1059) 2024-09-14T03:49:14.935 INFO:tasks.rados.rados.0.smithi110.stdout:3408: finishing copy_from racing read to smithi11075562-43 2024-09-14T03:49:14.935 INFO:tasks.rados.rados.0.smithi110.stdout:3408: finishing copy_from to smithi11075562-43 2024-09-14T03:49:14.935 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 2146 (ObjNum 1055 snap 332 seq_num 1055) dirty exists 2024-09-14T03:49:14.935 INFO:tasks.rados.rados.0.smithi110.stdout:3410: finishing copy_from racing read to smithi11075562-44 2024-09-14T03:49:14.935 INFO:tasks.rados.rados.0.smithi110.stdout:3410: finishing copy_from to smithi11075562-44 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 2143 (ObjNum 1008 snap 323 seq_num 1008) dirty exists 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3412: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3412: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1852 (ObjNum 33 snap 0 seq_num 6) dirty dne 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3411: finishing write tid 1 to smithi11075562-39 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3411: finishing write tid 2 to smithi11075562-39 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3411: finishing write tid 3 to smithi11075562-39 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3411: finishing write tid 4 to smithi11075562-39 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 2149 (ObjNum 1061 snap 335 seq_num 1061) dirty exists 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3411: left oid 39 (ObjNum 1061 snap 335 seq_num 1061) 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3408: done (4 left) 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3409: done (3 left) 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3410: done (2 left) 2024-09-14T03:49:14.936 INFO:tasks.rados.rados.0.smithi110.stdout:3411: done (1 left) 2024-09-14T03:49:14.937 INFO:tasks.rados.rados.0.smithi110.stdout:3412: done (0 left) 2024-09-14T03:49:14.937 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 335 2024-09-14T03:49:14.937 INFO:tasks.rados.rados.0.smithi110.stdout:3413: seq_num 1062 ranges {3817472=589824,4407296=401408} 2024-09-14T03:49:14.958 INFO:tasks.rados.rados.0.smithi110.stdout:3413: writing smithi11075562-28 from 3817472 to 4407296 tid 1 2024-09-14T03:49:14.960 INFO:tasks.rados.rados.0.smithi110.stdout:3413: writing smithi11075562-28 from 4407296 to 4808704 tid 2 2024-09-14T03:49:14.961 INFO:tasks.rados.rados.0.smithi110.stdout:3414: snap_create 2024-09-14T03:49:14.962 INFO:tasks.rados.rados.0.smithi110.stdout:3413: finishing write tid 1 to smithi11075562-28 2024-09-14T03:49:14.963 INFO:tasks.rados.rados.0.smithi110.stdout:3413: finishing write tid 2 to smithi11075562-28 2024-09-14T03:49:14.963 INFO:tasks.rados.rados.0.smithi110.stdout:3413: finishing write tid 3 to smithi11075562-28 2024-09-14T03:49:14.964 INFO:tasks.rados.rados.0.smithi110.stdout:3413: finishing write tid 4 to smithi11075562-28 2024-09-14T03:49:14.964 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 2005 (ObjNum 1062 snap 335 seq_num 1062) dirty exists 2024-09-14T03:49:14.964 INFO:tasks.rados.rados.0.smithi110.stdout:3413: left oid 28 (ObjNum 1062 snap 335 seq_num 1062) 2024-09-14T03:49:15.814 INFO:tasks.rados.rados.0.smithi110.stdout:3413: done (1 left) 2024-09-14T03:49:15.814 INFO:tasks.rados.rados.0.smithi110.stdout:3414: done (0 left) 2024-09-14T03:49:15.814 INFO:tasks.rados.rados.0.smithi110.stdout:3415: rmattr oid 26 current snap is 336 2024-09-14T03:49:15.814 INFO:tasks.rados.rados.0.smithi110.stdout:3415: done (0 left) 2024-09-14T03:49:15.814 INFO:tasks.rados.rados.0.smithi110.stdout:3416: rollback oid 26 current snap is 336 2024-09-14T03:49:15.814 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 332 2024-09-14T03:49:15.815 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 336 2024-09-14T03:49:15.815 INFO:tasks.rados.rados.0.smithi110.stdout:3417: seq_num 1063 ranges {2260992=606208,2867200=401408} 2024-09-14T03:49:15.829 INFO:tasks.rados.rados.0.smithi110.stdout:3417: writing smithi11075562-9 from 2260992 to 2867200 tid 1 2024-09-14T03:49:15.832 INFO:tasks.rados.rados.0.smithi110.stdout:3417: writing smithi11075562-9 from 2867200 to 3268608 tid 2 2024-09-14T03:49:15.832 INFO:tasks.rados.rados.0.smithi110.stdout:3418: snap_remove snap 331 2024-09-14T03:49:15.832 INFO:tasks.rados.rados.0.smithi110.stdout:3416: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:49:15.832 INFO:tasks.rados.rados.0.smithi110.stdout:3416: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:49:15.832 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1854 (ObjNum 1047 snap 330 seq_num 1047) dirty exists 2024-09-14T03:49:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:15 smithi110 ceph-mon[88262]: pgmap v750: 121 pgs: 121 active+clean; 244 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 644 KiB/s wr, 13 op/s 2024-09-14T03:49:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:15 smithi110 ceph-mon[88262]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T03:49:16.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:15 smithi110 ceph-mon[86423]: pgmap v750: 121 pgs: 121 active+clean; 244 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 644 KiB/s wr, 13 op/s 2024-09-14T03:49:16.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:15 smithi110 ceph-mon[86423]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T03:49:16.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:15 smithi195 ceph-mon[82388]: pgmap v750: 121 pgs: 121 active+clean; 244 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 644 KiB/s wr, 13 op/s 2024-09-14T03:49:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:15 smithi195 ceph-mon[82388]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T03:49:16.821 INFO:tasks.rados.rados.0.smithi110.stdout:3416: done (2 left) 2024-09-14T03:49:16.821 INFO:tasks.rados.rados.0.smithi110.stdout:3418: done (1 left) 2024-09-14T03:49:16.821 INFO:tasks.rados.rados.0.smithi110.stdout:3419: delete oid 50 current snap is 336 2024-09-14T03:49:16.821 INFO:tasks.rados.rados.0.smithi110.stdout:3417: finishing write tid 1 to smithi11075562-9 2024-09-14T03:49:16.821 INFO:tasks.rados.rados.0.smithi110.stdout:3417: finishing write tid 2 to smithi11075562-9 2024-09-14T03:49:16.821 INFO:tasks.rados.rados.0.smithi110.stdout:3417: finishing write tid 3 to smithi11075562-9 2024-09-14T03:49:16.821 INFO:tasks.rados.rados.0.smithi110.stdout:3417: finishing write tid 4 to smithi11075562-9 2024-09-14T03:49:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 2137 (ObjNum 1063 snap 336 seq_num 1063) dirty exists 2024-09-14T03:49:16.822 INFO:tasks.rados.rados.0.smithi110.stdout:3417: left oid 9 (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:49:16.829 INFO:tasks.rados.rados.0.smithi110.stdout:3417: done (1 left) 2024-09-14T03:49:16.829 INFO:tasks.rados.rados.0.smithi110.stdout:3419: done (0 left) 2024-09-14T03:49:16.829 INFO:tasks.rados.rados.0.smithi110.stdout:3420: copy_from oid 26 from oid 30 current snap is 336 2024-09-14T03:49:16.829 INFO:tasks.rados.rados.0.smithi110.stdout:3421: snap_remove snap 327 2024-09-14T03:49:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:16 smithi110 ceph-mon[88262]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T03:49:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:16 smithi110 ceph-mon[86423]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T03:49:17.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:16 smithi195 ceph-mon[82388]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T03:49:17.825 INFO:tasks.rados.rados.0.smithi110.stdout:3421: done (1 left) 2024-09-14T03:49:17.825 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 336 2024-09-14T03:49:17.825 INFO:tasks.rados.rados.0.smithi110.stdout:3422: seq_num 1065 ranges {7036928=638976,7675904=401408} 2024-09-14T03:49:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:3422: writing smithi11075562-23 from 7036928 to 7675904 tid 1 2024-09-14T03:49:17.869 INFO:tasks.rados.rados.0.smithi110.stdout:3422: writing smithi11075562-23 from 7675904 to 8077312 tid 2 2024-09-14T03:49:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:3423: rmattr oid 8 current snap is 336 2024-09-14T03:49:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:3423: done (2 left) 2024-09-14T03:49:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 336 2024-09-14T03:49:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:3420: finishing copy_from racing read to smithi11075562-26 2024-09-14T03:49:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:3420: finishing copy_from to smithi11075562-26 2024-09-14T03:49:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 1997 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:49:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:3424: seq_num 1066 ranges {0=663552,663552=401408} 2024-09-14T03:49:17.873 INFO:tasks.rados.rados.0.smithi110.stdout:3424: writing smithi11075562-49 from 0 to 663552 tid 1 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:3424: writing smithi11075562-49 from 663552 to 1064960 tid 2 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:3420: done (2 left) 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 336 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:3422: finishing write tid 1 to smithi11075562-23 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:3422: finishing write tid 2 to smithi11075562-23 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:3422: finishing write tid 3 to smithi11075562-23 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:3422: finishing write tid 4 to smithi11075562-23 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 2021 (ObjNum 1065 snap 336 seq_num 1065) dirty exists 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:3422: left oid 23 (ObjNum 1065 snap 336 seq_num 1065) 2024-09-14T03:49:17.876 INFO:tasks.rados.rados.0.smithi110.stdout:3425: seq_num 1067 ranges {5562368=679936,6242304=401408} 2024-09-14T03:49:17.908 INFO:tasks.rados.rados.0.smithi110.stdout:3425: writing smithi11075562-10 from 5562368 to 6242304 tid 1 2024-09-14T03:49:17.911 INFO:tasks.rados.rados.0.smithi110.stdout:3425: writing smithi11075562-10 from 6242304 to 6643712 tid 2 2024-09-14T03:49:17.911 INFO:tasks.rados.rados.0.smithi110.stdout:3422: done (2 left) 2024-09-14T03:49:17.911 INFO:tasks.rados.rados.0.smithi110.stdout:3426: snap_remove snap 326 2024-09-14T03:49:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:17 smithi110 ceph-mon[86423]: pgmap v753: 121 pgs: 121 active+clean; 246 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-14T03:49:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:17 smithi110 ceph-mon[86423]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T03:49:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:17 smithi110 ceph-mon[88262]: pgmap v753: 121 pgs: 121 active+clean; 246 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-14T03:49:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:17 smithi110 ceph-mon[88262]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T03:49:18.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:17 smithi195 ceph-mon[82388]: pgmap v753: 121 pgs: 121 active+clean; 246 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-14T03:49:18.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:17 smithi195 ceph-mon[82388]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T03:49:18.805 INFO:tasks.rados.rados.0.smithi110.stdout:3424: finishing write tid 1 to smithi11075562-49 2024-09-14T03:49:18.805 INFO:tasks.rados.rados.0.smithi110.stdout:3424: finishing write tid 2 to smithi11075562-49 2024-09-14T03:49:18.805 INFO:tasks.rados.rados.0.smithi110.stdout:3424: finishing write tid 3 to smithi11075562-49 2024-09-14T03:49:18.805 INFO:tasks.rados.rados.0.smithi110.stdout:3424: finishing write tid 4 to smithi11075562-49 2024-09-14T03:49:18.806 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2119 (ObjNum 1066 snap 336 seq_num 1066) dirty exists 2024-09-14T03:49:18.806 INFO:tasks.rados.rados.0.smithi110.stdout:3424: left oid 49 (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:49:18.806 INFO:tasks.rados.rados.0.smithi110.stdout:3425: finishing write tid 1 to smithi11075562-10 2024-09-14T03:49:18.806 INFO:tasks.rados.rados.0.smithi110.stdout:3425: finishing write tid 2 to smithi11075562-10 2024-09-14T03:49:18.806 INFO:tasks.rados.rados.0.smithi110.stdout:3425: finishing write tid 3 to smithi11075562-10 2024-09-14T03:49:18.806 INFO:tasks.rados.rados.0.smithi110.stdout:3425: finishing write tid 4 to smithi11075562-10 2024-09-14T03:49:18.806 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2024 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:49:18.806 INFO:tasks.rados.rados.0.smithi110.stdout:3425: left oid 10 (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:49:18.807 INFO:tasks.rados.rados.0.smithi110.stdout:3424: done (2 left) 2024-09-14T03:49:18.807 INFO:tasks.rados.rados.0.smithi110.stdout:3425: done (1 left) 2024-09-14T03:49:18.807 INFO:tasks.rados.rados.0.smithi110.stdout:3426: done (0 left) 2024-09-14T03:49:18.807 INFO:tasks.rados.rados.0.smithi110.stdout:3427: snap_create 2024-09-14T03:49:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:18 smithi110 ceph-mon[88262]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T03:49:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:18 smithi110 ceph-mon[88262]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T03:49:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:18 smithi110 ceph-mon[86423]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T03:49:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:18 smithi110 ceph-mon[86423]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T03:49:19.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:18 smithi195 ceph-mon[82388]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T03:49:19.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:18 smithi195 ceph-mon[82388]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T03:49:19.806 INFO:tasks.rados.rados.0.smithi110.stdout:3427: done (0 left) 2024-09-14T03:49:19.806 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 337 2024-09-14T03:49:19.806 INFO:tasks.rados.rados.0.smithi110.stdout:3428: seq_num 1068 ranges {5234688=696320,5931008=401408} 2024-09-14T03:49:19.838 INFO:tasks.rados.rados.0.smithi110.stdout:3428: writing smithi11075562-17 from 5234688 to 5931008 tid 1 2024-09-14T03:49:19.840 INFO:tasks.rados.rados.0.smithi110.stdout:3428: writing smithi11075562-17 from 5931008 to 6332416 tid 2 2024-09-14T03:49:19.840 INFO:tasks.rados.rados.0.smithi110.stdout:3429: read oid 7 snap -1 2024-09-14T03:49:19.840 INFO:tasks.rados.rados.0.smithi110.stdout:3429: expect (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:49:19.840 INFO:tasks.rados.rados.0.smithi110.stdout:3430: snap_remove snap 336 2024-09-14T03:49:20.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:19 smithi110 ceph-mon[88262]: pgmap v756: 121 pgs: 121 active+clean; 249 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-14T03:49:20.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:19 smithi110 ceph-mon[88262]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T03:49:20.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:19 smithi110 ceph-mon[86423]: pgmap v756: 121 pgs: 121 active+clean; 249 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-14T03:49:20.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:19 smithi110 ceph-mon[86423]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T03:49:20.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:19 smithi195 ceph-mon[82388]: pgmap v756: 121 pgs: 121 active+clean; 249 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-14T03:49:20.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:19 smithi195 ceph-mon[82388]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T03:49:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:49:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:49:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:49:20.836 INFO:tasks.rados.rados.0.smithi110.stdout:3428: finishing write tid 1 to smithi11075562-17 2024-09-14T03:49:20.836 INFO:tasks.rados.rados.0.smithi110.stdout:3428: finishing write tid 2 to smithi11075562-17 2024-09-14T03:49:20.836 INFO:tasks.rados.rados.0.smithi110.stdout:3428: finishing write tid 3 to smithi11075562-17 2024-09-14T03:49:20.836 INFO:tasks.rados.rados.0.smithi110.stdout:3429: expect (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:49:20.862 INFO:tasks.rados.rados.0.smithi110.stdout:3428: finishing write tid 4 to smithi11075562-17 2024-09-14T03:49:20.862 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 2007 (ObjNum 1068 snap 337 seq_num 1068) dirty exists 2024-09-14T03:49:20.862 INFO:tasks.rados.rados.0.smithi110.stdout:3428: left oid 17 (ObjNum 1068 snap 337 seq_num 1068) 2024-09-14T03:49:20.862 INFO:tasks.rados.rados.0.smithi110.stdout:3428: done (2 left) 2024-09-14T03:49:20.862 INFO:tasks.rados.rados.0.smithi110.stdout:3429: done (1 left) 2024-09-14T03:49:20.862 INFO:tasks.rados.rados.0.smithi110.stdout:3430: done (0 left) 2024-09-14T03:49:20.862 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 337 2024-09-14T03:49:20.862 INFO:tasks.rados.rados.0.smithi110.stdout:3431: seq_num 1069 ranges {8249344=712704,8962048=401408} 2024-09-14T03:49:20.908 INFO:tasks.rados.rados.0.smithi110.stdout:3431: writing smithi11075562-4 from 8249344 to 8962048 tid 1 2024-09-14T03:49:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3431: writing smithi11075562-4 from 8962048 to 9363456 tid 2 2024-09-14T03:49:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3432: rmattr oid 39 current snap is 337 2024-09-14T03:49:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 337 2024-09-14T03:49:20.911 INFO:tasks.rados.rados.0.smithi110.stdout:3433: seq_num 1070 ranges {1769472=729088,2498560=401408} 2024-09-14T03:49:20.924 INFO:tasks.rados.rados.0.smithi110.stdout:3433: writing smithi11075562-3 from 1769472 to 2498560 tid 1 2024-09-14T03:49:20.927 INFO:tasks.rados.rados.0.smithi110.stdout:3433: writing smithi11075562-3 from 2498560 to 2899968 tid 2 2024-09-14T03:49:20.927 INFO:tasks.rados.rados.0.smithi110.stdout:3434: delete oid 34 current snap is 337 2024-09-14T03:49:20.927 INFO:tasks.rados.rados.0.smithi110.stdout:3431: finishing write tid 1 to smithi11075562-4 2024-09-14T03:49:20.928 INFO:tasks.rados.rados.0.smithi110.stdout:3431: finishing write tid 2 to smithi11075562-4 2024-09-14T03:49:20.928 INFO:tasks.rados.rados.0.smithi110.stdout:3431: finishing write tid 3 to smithi11075562-4 2024-09-14T03:49:20.928 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 2150 (ObjNum 1061 snap 335 seq_num 1061) dirty exists 2024-09-14T03:49:20.928 INFO:tasks.rados.rados.0.smithi110.stdout:3431: finishing write tid 4 to smithi11075562-4 2024-09-14T03:49:20.928 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2150 (ObjNum 1069 snap 337 seq_num 1069) dirty exists 2024-09-14T03:49:20.928 INFO:tasks.rados.rados.0.smithi110.stdout:3431: left oid 4 (ObjNum 1069 snap 337 seq_num 1069) 2024-09-14T03:49:20.929 INFO:tasks.rados.rados.0.smithi110.stdout:3431: done (3 left) 2024-09-14T03:49:20.929 INFO:tasks.rados.rados.0.smithi110.stdout:3432: done (2 left) 2024-09-14T03:49:20.929 INFO:tasks.rados.rados.0.smithi110.stdout:3434: done (1 left) 2024-09-14T03:49:20.930 INFO:tasks.rados.rados.0.smithi110.stdout:3435: copy_from oid 35 from oid 7 current snap is 337 2024-09-14T03:49:20.930 INFO:tasks.rados.rados.0.smithi110.stdout:3436: rmattr oid 43 current snap is 337 2024-09-14T03:49:20.930 INFO:tasks.rados.rados.0.smithi110.stdout:3437: snap_remove snap 325 2024-09-14T03:49:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:20 smithi110 ceph-mon[88262]: pgmap v759: 121 pgs: 121 active+clean; 261 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.8 MiB/s wr, 6 op/s 2024-09-14T03:49:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:20 smithi110 ceph-mon[86423]: pgmap v759: 121 pgs: 121 active+clean; 261 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.8 MiB/s wr, 6 op/s 2024-09-14T03:49:21.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:20 smithi195 ceph-mon[82388]: pgmap v759: 121 pgs: 121 active+clean; 261 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.8 MiB/s wr, 6 op/s 2024-09-14T03:49:21.833 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:49:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:49:21] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-14T03:49:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:3437: done (3 left) 2024-09-14T03:49:21.842 INFO:tasks.rados.rados.0.smithi110.stdout:3438: setattr oid 34 current snap is 337 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:3433: finishing write tid 1 to smithi11075562-3 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:3433: finishing write tid 2 to smithi11075562-3 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:3433: finishing write tid 3 to smithi11075562-3 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:3433: finishing write tid 4 to smithi11075562-3 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 2008 (ObjNum 1070 snap 337 seq_num 1070) dirty exists 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:3433: left oid 3 (ObjNum 1070 snap 337 seq_num 1070) 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 2151 (ObjNum 1055 snap 332 seq_num 1055) dirty exists 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:3435: finishing copy_from racing read to smithi11075562-35 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:3435: finishing copy_from to smithi11075562-35 2024-09-14T03:49:21.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 1636 (ObjNum 1060 snap 335 seq_num 1060) dirty exists 2024-09-14T03:49:21.847 INFO:tasks.rados.rados.0.smithi110.stdout:3433: done (3 left) 2024-09-14T03:49:21.847 INFO:tasks.rados.rados.0.smithi110.stdout:3435: done (2 left) 2024-09-14T03:49:21.847 INFO:tasks.rados.rados.0.smithi110.stdout:3436: done (1 left) 2024-09-14T03:49:21.847 INFO:tasks.rados.rados.0.smithi110.stdout:3439: copy_from oid 3 from oid 9 current snap is 337 2024-09-14T03:49:21.847 INFO:tasks.rados.rados.0.smithi110.stdout:3440: read oid 44 snap -1 2024-09-14T03:49:21.847 INFO:tasks.rados.rados.0.smithi110.stdout:3440: expect (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:49:21.847 INFO:tasks.rados.rados.0.smithi110.stdout:3441: snap_remove snap 329 2024-09-14T03:49:22.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:21 smithi110 ceph-mon[88262]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T03:49:22.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:21 smithi110 ceph-mon[86423]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T03:49:22.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:21 smithi195 ceph-mon[82388]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T03:49:22.853 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 2152 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:49:22.854 INFO:tasks.rados.rados.0.smithi110.stdout:3439: finishing copy_from racing read to smithi11075562-3 2024-09-14T03:49:22.854 INFO:tasks.rados.rados.0.smithi110.stdout:3439: finishing copy_from to smithi11075562-3 2024-09-14T03:49:22.854 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 2137 (ObjNum 1063 snap 336 seq_num 1063) dirty exists 2024-09-14T03:49:22.854 INFO:tasks.rados.rados.0.smithi110.stdout:3440: expect (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:49:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:22 smithi110 ceph-mon[88262]: pgmap v761: 121 pgs: 121 active+clean; 264 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.8 MiB/s wr, 6 op/s 2024-09-14T03:49:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:22 smithi110 ceph-mon[88262]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T03:49:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:22 smithi110 ceph-mon[86423]: pgmap v761: 121 pgs: 121 active+clean; 264 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.8 MiB/s wr, 6 op/s 2024-09-14T03:49:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:22 smithi110 ceph-mon[86423]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T03:49:23.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:22 smithi195 ceph-mon[82388]: pgmap v761: 121 pgs: 121 active+clean; 264 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.8 MiB/s wr, 6 op/s 2024-09-14T03:49:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:22 smithi195 ceph-mon[82388]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T03:49:23.435 INFO:tasks.rados.rados.0.smithi110.stdout:3438: done (3 left) 2024-09-14T03:49:23.436 INFO:tasks.rados.rados.0.smithi110.stdout:3439: done (2 left) 2024-09-14T03:49:23.436 INFO:tasks.rados.rados.0.smithi110.stdout:3440: done (1 left) 2024-09-14T03:49:23.436 INFO:tasks.rados.rados.0.smithi110.stdout:3441: done (0 left) 2024-09-14T03:49:23.436 INFO:tasks.rados.rados.0.smithi110.stdout:3442: snap_remove snap 332 2024-09-14T03:49:23.805 INFO:tasks.rados.rados.0.smithi110.stdout:3442: done (0 left) 2024-09-14T03:49:23.805 INFO:tasks.rados.rados.0.smithi110.stdout:3443: snap_remove snap 324 2024-09-14T03:49:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:23 smithi110 ceph-mon[88262]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T03:49:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:23 smithi110 ceph-mon[88262]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T03:49:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:23 smithi110 ceph-mon[86423]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T03:49:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:23 smithi110 ceph-mon[86423]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T03:49:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:23 smithi195 ceph-mon[82388]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T03:49:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:23 smithi195 ceph-mon[82388]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T03:49:24.814 INFO:tasks.rados.rados.0.smithi110.stdout:3443: done (0 left) 2024-09-14T03:49:24.814 INFO:tasks.rados.rados.0.smithi110.stdout:3444: read oid 32 snap -1 2024-09-14T03:49:24.814 INFO:tasks.rados.rados.0.smithi110.stdout:3444: expect (ObjNum 1048 snap 330 seq_num 1048) 2024-09-14T03:49:24.814 INFO:tasks.rados.rados.0.smithi110.stdout:3445: read oid 35 snap -1 2024-09-14T03:49:24.814 INFO:tasks.rados.rados.0.smithi110.stdout:3445: expect (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:49:24.814 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 337 2024-09-14T03:49:24.814 INFO:tasks.rados.rados.0.smithi110.stdout:3446: seq_num 1072 ranges {9551872=761856,10313728=401408} 2024-09-14T03:49:24.871 INFO:tasks.rados.rados.0.smithi110.stdout:3446: writing smithi11075562-5 from 9551872 to 10313728 tid 1 2024-09-14T03:49:24.873 INFO:tasks.rados.rados.0.smithi110.stdout:3446: writing smithi11075562-5 from 10313728 to 10715136 tid 2 2024-09-14T03:49:24.874 INFO:tasks.rados.rados.0.smithi110.stdout:3447: delete oid 50 current snap is 337 2024-09-14T03:49:24.874 INFO:tasks.rados.rados.0.smithi110.stdout:3444: expect (ObjNum 1048 snap 330 seq_num 1048) 2024-09-14T03:49:24.893 INFO:tasks.rados.rados.0.smithi110.stdout:3445: expect (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3446: finishing write tid 1 to smithi11075562-5 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3446: finishing write tid 2 to smithi11075562-5 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3446: finishing write tid 3 to smithi11075562-5 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3446: finishing write tid 4 to smithi11075562-5 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2427 (ObjNum 1072 snap 337 seq_num 1072) dirty exists 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3446: left oid 5 (ObjNum 1072 snap 337 seq_num 1072) 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3444: done (3 left) 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3445: done (2 left) 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3446: done (1 left) 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3447: done (0 left) 2024-09-14T03:49:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3448: read oid 10 snap 316 2024-09-14T03:49:24.919 INFO:tasks.rados.rados.0.smithi110.stdout:3448: expect (ObjNum 983 snap 315 seq_num 983) 2024-09-14T03:49:24.919 INFO:tasks.rados.rados.0.smithi110.stdout:3449: delete oid 32 current snap is 337 2024-09-14T03:49:24.920 INFO:tasks.rados.rados.0.smithi110.stdout:3449: done (1 left) 2024-09-14T03:49:24.920 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 337 2024-09-14T03:49:24.920 INFO:tasks.rados.rados.0.smithi110.stdout:3450: seq_num 1075 ranges {0=409600,409600=671744,1081344=131072} 2024-09-14T03:49:24.923 INFO:tasks.rados.rados.0.smithi110.stdout:3450: writing smithi11075562-32 from 0 to 409600 tid 1 2024-09-14T03:49:24.927 INFO:tasks.rados.rados.0.smithi110.stdout:3450: writing smithi11075562-32 from 409600 to 1081344 tid 2 2024-09-14T03:49:24.928 INFO:tasks.rados.rados.0.smithi110.stdout:3450: writing smithi11075562-32 from 1081344 to 1212416 tid 3 2024-09-14T03:49:24.928 INFO:tasks.rados.rados.0.smithi110.stdout:3451: read oid 31 snap -1 2024-09-14T03:49:24.928 INFO:tasks.rados.rados.0.smithi110.stdout:3451: expect deleted 2024-09-14T03:49:24.929 INFO:tasks.rados.rados.0.smithi110.stdout:3452: copy_from oid 2 from oid 49 current snap is 337 2024-09-14T03:49:24.929 INFO:tasks.rados.rados.0.smithi110.stdout:3453: snap_remove snap 335 2024-09-14T03:49:25.818 INFO:tasks.rados.rados.0.smithi110.stdout:3453: done (4 left) 2024-09-14T03:49:25.819 INFO:tasks.rados.rados.0.smithi110.stdout:3454: snap_create 2024-09-14T03:49:25.819 INFO:tasks.rados.rados.0.smithi110.stdout:3448: expect (ObjNum 983 snap 315 seq_num 983) 2024-09-14T03:49:25.976 INFO:tasks.rados.rados.0.smithi110.stdout:3450: finishing write tid 1 to smithi11075562-32 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:3450: finishing write tid 2 to smithi11075562-32 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:3450: finishing write tid 3 to smithi11075562-32 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:3450: finishing write tid 4 to smithi11075562-32 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:3452: finishing copy_from racing read to smithi11075562-2 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:3452: finishing copy_from to smithi11075562-2 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 2119 (ObjNum 1066 snap 336 seq_num 1066) dirty exists 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:3450: finishing write tid 5 to smithi11075562-32 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2051 (ObjNum 1075 snap 337 seq_num 1075) dirty exists 2024-09-14T03:49:25.977 INFO:tasks.rados.rados.0.smithi110.stdout:3450: left oid 32 (ObjNum 1075 snap 337 seq_num 1075) 2024-09-14T03:49:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:25 smithi110 ceph-mon[88262]: pgmap v765: 121 pgs: 121 active+clean; 264 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 930 KiB/s wr, 6 op/s 2024-09-14T03:49:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:25 smithi110 ceph-mon[88262]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T03:49:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:25 smithi110 ceph-mon[86423]: pgmap v765: 121 pgs: 121 active+clean; 264 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 930 KiB/s wr, 6 op/s 2024-09-14T03:49:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:25 smithi110 ceph-mon[86423]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T03:49:26.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:25 smithi195 ceph-mon[82388]: pgmap v765: 121 pgs: 121 active+clean; 264 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 930 KiB/s wr, 6 op/s 2024-09-14T03:49:26.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:25 smithi195 ceph-mon[82388]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T03:49:26.824 INFO:tasks.rados.rados.0.smithi110.stdout:3448: done (4 left) 2024-09-14T03:49:26.824 INFO:tasks.rados.rados.0.smithi110.stdout:3450: done (3 left) 2024-09-14T03:49:26.824 INFO:tasks.rados.rados.0.smithi110.stdout:3451: done (2 left) 2024-09-14T03:49:26.824 INFO:tasks.rados.rados.0.smithi110.stdout:3452: done (1 left) 2024-09-14T03:49:26.825 INFO:tasks.rados.rados.0.smithi110.stdout:3454: done (0 left) 2024-09-14T03:49:26.825 INFO:tasks.rados.rados.0.smithi110.stdout:3455: rmattr oid 21 current snap is 338 2024-09-14T03:49:26.825 INFO:tasks.rados.rados.0.smithi110.stdout:3456: snap_remove snap 337 2024-09-14T03:49:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:26 smithi110 ceph-mon[86423]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T03:49:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:26 smithi110 ceph-mon[88262]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T03:49:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:26 smithi195 ceph-mon[82388]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T03:49:27.826 INFO:tasks.rados.rados.0.smithi110.stdout:3456: done (1 left) 2024-09-14T03:49:27.826 INFO:tasks.rados.rados.0.smithi110.stdout:3457: rmattr oid 15 current snap is 338 2024-09-14T03:49:27.826 INFO:tasks.rados.rados.0.smithi110.stdout:3457: done (1 left) 2024-09-14T03:49:27.826 INFO:tasks.rados.rados.0.smithi110.stdout:3458: setattr oid 4 current snap is 338 2024-09-14T03:49:27.826 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2152 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:49:27.828 INFO:tasks.rados.rados.0.smithi110.stdout:3455: done (1 left) 2024-09-14T03:49:27.829 INFO:tasks.rados.rados.0.smithi110.stdout:3459: read oid 11 snap 316 2024-09-14T03:49:27.829 INFO:tasks.rados.rados.0.smithi110.stdout:3459: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:49:27.829 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 338 2024-09-14T03:49:27.829 INFO:tasks.rados.rados.0.smithi110.stdout:3460: seq_num 1076 ranges {5136384=425984,5562368=548864,6111232=253952} 2024-09-14T03:49:27.859 INFO:tasks.rados.rados.0.smithi110.stdout:3460: writing smithi11075562-36 from 5136384 to 5562368 tid 1 2024-09-14T03:49:27.862 INFO:tasks.rados.rados.0.smithi110.stdout:3460: writing smithi11075562-36 from 5562368 to 6111232 tid 2 2024-09-14T03:49:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:3460: writing smithi11075562-36 from 6111232 to 6365184 tid 3 2024-09-14T03:49:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:3461: snap_create 2024-09-14T03:49:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2153 (ObjNum 1069 snap 337 seq_num 1069) dirty exists 2024-09-14T03:49:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:3459: expect (ObjNum 925 snap 301 seq_num 925) 2024-09-14T03:49:27.932 INFO:tasks.rados.rados.0.smithi110.stdout:3460: finishing write tid 1 to smithi11075562-36 2024-09-14T03:49:27.933 INFO:tasks.rados.rados.0.smithi110.stdout:3460: finishing write tid 2 to smithi11075562-36 2024-09-14T03:49:27.933 INFO:tasks.rados.rados.0.smithi110.stdout:3460: finishing write tid 3 to smithi11075562-36 2024-09-14T03:49:27.933 INFO:tasks.rados.rados.0.smithi110.stdout:3460: finishing write tid 4 to smithi11075562-36 2024-09-14T03:49:27.933 INFO:tasks.rados.rados.0.smithi110.stdout:3460: finishing write tid 5 to smithi11075562-36 2024-09-14T03:49:27.933 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2433 (ObjNum 1076 snap 338 seq_num 1076) dirty exists 2024-09-14T03:49:27.933 INFO:tasks.rados.rados.0.smithi110.stdout:3460: left oid 36 (ObjNum 1076 snap 338 seq_num 1076) 2024-09-14T03:49:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:27 smithi110 ceph-mon[88262]: pgmap v768: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 260 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-09-14T03:49:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:27 smithi110 ceph-mon[88262]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T03:49:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:27 smithi110 ceph-mon[86423]: pgmap v768: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 260 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-09-14T03:49:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:27 smithi110 ceph-mon[86423]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T03:49:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:28.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:27 smithi195 ceph-mon[82388]: pgmap v768: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 260 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-09-14T03:49:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:27 smithi195 ceph-mon[82388]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T03:49:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:28.808 INFO:tasks.rados.rados.0.smithi110.stdout:3458: done (3 left) 2024-09-14T03:49:28.808 INFO:tasks.rados.rados.0.smithi110.stdout:3459: done (2 left) 2024-09-14T03:49:28.808 INFO:tasks.rados.rados.0.smithi110.stdout:3460: done (1 left) 2024-09-14T03:49:28.808 INFO:tasks.rados.rados.0.smithi110.stdout:3461: done (0 left) 2024-09-14T03:49:28.808 INFO:tasks.rados.rados.0.smithi110.stdout:3462: snap_remove snap 338 2024-09-14T03:49:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:28 smithi110 ceph-mon[88262]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T03:49:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:28 smithi110 ceph-mon[88262]: pgmap v771: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 252 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-14T03:49:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:28 smithi110 ceph-mon[88262]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T03:49:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:28 smithi110 ceph-mon[86423]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T03:49:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:28 smithi110 ceph-mon[86423]: pgmap v771: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 252 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-14T03:49:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:28 smithi110 ceph-mon[86423]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T03:49:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:28 smithi195 ceph-mon[82388]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T03:49:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:28 smithi195 ceph-mon[82388]: pgmap v771: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 252 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-14T03:49:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:28 smithi195 ceph-mon[82388]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T03:49:29.810 INFO:tasks.rados.rados.0.smithi110.stdout:3462: done (0 left) 2024-09-14T03:49:29.810 INFO:tasks.rados.rados.0.smithi110.stdout:3463: snap_create 2024-09-14T03:49:30.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:49:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:49:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:49:30.815 INFO:tasks.rados.rados.0.smithi110.stdout:3463: done (0 left) 2024-09-14T03:49:30.815 INFO:tasks.rados.rados.0.smithi110.stdout:3464: read oid 11 snap -1 2024-09-14T03:49:30.815 INFO:tasks.rados.rados.0.smithi110.stdout:3464: expect (ObjNum 1056 snap 332 seq_num 1056) 2024-09-14T03:49:30.815 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 340 2024-09-14T03:49:30.816 INFO:tasks.rados.rados.0.smithi110.stdout:3465: seq_num 1077 ranges {4808704=442368,5251072=434176,5685248=368640} 2024-09-14T03:49:30.844 INFO:tasks.rados.rados.0.smithi110.stdout:3465: writing smithi11075562-28 from 4808704 to 5251072 tid 1 2024-09-14T03:49:30.846 INFO:tasks.rados.rados.0.smithi110.stdout:3465: writing smithi11075562-28 from 5251072 to 5685248 tid 2 2024-09-14T03:49:30.849 INFO:tasks.rados.rados.0.smithi110.stdout:3465: writing smithi11075562-28 from 5685248 to 6053888 tid 3 2024-09-14T03:49:30.849 INFO:tasks.rados.rados.0.smithi110.stdout:3466: delete oid 46 current snap is 340 2024-09-14T03:49:30.849 INFO:tasks.rados.rados.0.smithi110.stdout:3464: expect (ObjNum 1056 snap 332 seq_num 1056) 2024-09-14T03:49:30.974 INFO:tasks.rados.rados.0.smithi110.stdout:3465: finishing write tid 1 to smithi11075562-28 2024-09-14T03:49:30.974 INFO:tasks.rados.rados.0.smithi110.stdout:3465: finishing write tid 2 to smithi11075562-28 2024-09-14T03:49:30.974 INFO:tasks.rados.rados.0.smithi110.stdout:3465: finishing write tid 3 to smithi11075562-28 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3465: finishing write tid 4 to smithi11075562-28 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3465: finishing write tid 5 to smithi11075562-28 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 2141 (ObjNum 1077 snap 340 seq_num 1077) dirty exists 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3465: left oid 28 (ObjNum 1077 snap 340 seq_num 1077) 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3464: done (2 left) 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3465: done (1 left) 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3466: done (0 left) 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3467: read oid 9 snap -1 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3467: expect (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3468: copy_from oid 45 from oid 27 current snap is 340 2024-09-14T03:49:30.975 INFO:tasks.rados.rados.0.smithi110.stdout:3469: rollback oid 49 current snap is 340 2024-09-14T03:49:30.976 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 339 2024-09-14T03:49:30.976 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 340 2024-09-14T03:49:30.976 INFO:tasks.rados.rados.0.smithi110.stdout:3470: seq_num 1079 ranges {942080=475136,1417216=606208,2023424=196608} 2024-09-14T03:49:30.981 INFO:tasks.rados.rados.0.smithi110.stdout:3470: writing smithi11075562-12 from 942080 to 1417216 tid 1 2024-09-14T03:49:30.985 INFO:tasks.rados.rados.0.smithi110.stdout:3470: writing smithi11075562-12 from 1417216 to 2023424 tid 2 2024-09-14T03:49:30.986 INFO:tasks.rados.rados.0.smithi110.stdout:3470: writing smithi11075562-12 from 2023424 to 2220032 tid 3 2024-09-14T03:49:30.986 INFO:tasks.rados.rados.0.smithi110.stdout:3471: read oid 3 snap -1 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:3471: expect (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:3472: setattr oid 32 current snap is 340 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:3468: finishing copy_from to smithi11075562-45 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:3468: got expected ENOENT (src dne) 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:3468: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:3469: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:3469: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:3469: finishing rollback tid 2 to smithi11075562-49 2024-09-14T03:49:30.987 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2161 (ObjNum 1066 snap 336 seq_num 1066) dirty exists 2024-09-14T03:49:30.988 INFO:tasks.rados.rados.0.smithi110.stdout:3468: done (5 left) 2024-09-14T03:49:30.988 INFO:tasks.rados.rados.0.smithi110.stdout:3469: done (4 left) 2024-09-14T03:49:30.988 INFO:tasks.rados.rados.0.smithi110.stdout:3473: snap_create 2024-09-14T03:49:30.991 INFO:tasks.rados.rados.0.smithi110.stdout:3467: expect (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:49:31.113 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2052 (ObjNum 1075 snap 337 seq_num 1075) dirty exists 2024-09-14T03:49:31.113 INFO:tasks.rados.rados.0.smithi110.stdout:3470: finishing write tid 1 to smithi11075562-12 2024-09-14T03:49:31.113 INFO:tasks.rados.rados.0.smithi110.stdout:3470: finishing write tid 2 to smithi11075562-12 2024-09-14T03:49:31.114 INFO:tasks.rados.rados.0.smithi110.stdout:3470: finishing write tid 3 to smithi11075562-12 2024-09-14T03:49:31.114 INFO:tasks.rados.rados.0.smithi110.stdout:3470: finishing write tid 4 to smithi11075562-12 2024-09-14T03:49:31.114 INFO:tasks.rados.rados.0.smithi110.stdout:3470: finishing write tid 5 to smithi11075562-12 2024-09-14T03:49:31.114 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2438 (ObjNum 1079 snap 340 seq_num 1079) dirty exists 2024-09-14T03:49:31.114 INFO:tasks.rados.rados.0.smithi110.stdout:3470: left oid 12 (ObjNum 1079 snap 340 seq_num 1079) 2024-09-14T03:49:31.114 INFO:tasks.rados.rados.0.smithi110.stdout:3471: expect (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:49:31.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:30 smithi110 ceph-mon[88262]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T03:49:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:30 smithi110 ceph-mon[86423]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T03:49:31.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:30 smithi195 ceph-mon[82388]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T03:49:31.818 INFO:tasks.rados.rados.0.smithi110.stdout:3467: done (4 left) 2024-09-14T03:49:31.818 INFO:tasks.rados.rados.0.smithi110.stdout:3470: done (3 left) 2024-09-14T03:49:31.818 INFO:tasks.rados.rados.0.smithi110.stdout:3471: done (2 left) 2024-09-14T03:49:31.818 INFO:tasks.rados.rados.0.smithi110.stdout:3472: done (1 left) 2024-09-14T03:49:31.818 INFO:tasks.rados.rados.0.smithi110.stdout:3473: done (0 left) 2024-09-14T03:49:31.819 INFO:tasks.rados.rados.0.smithi110.stdout:3474: snap_create 2024-09-14T03:49:31.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:31 smithi110 ceph-mon[88262]: pgmap v774: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 260 KiB/s wr, 4 op/s 2024-09-14T03:49:31.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:31 smithi110 ceph-mon[88262]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T03:49:31.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:31 smithi110 ceph-mon[86423]: pgmap v774: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 260 KiB/s wr, 4 op/s 2024-09-14T03:49:31.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:31 smithi110 ceph-mon[86423]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T03:49:31.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:49:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:49:31] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-14T03:49:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:31 smithi195 ceph-mon[82388]: pgmap v774: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 260 KiB/s wr, 4 op/s 2024-09-14T03:49:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:31 smithi195 ceph-mon[82388]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T03:49:32.824 INFO:tasks.rados.rados.0.smithi110.stdout:3474: done (0 left) 2024-09-14T03:49:32.824 INFO:tasks.rados.rados.0.smithi110.stdout:3475: read oid 29 snap -1 2024-09-14T03:49:32.825 INFO:tasks.rados.rados.0.smithi110.stdout:3475: expect deleted 2024-09-14T03:49:32.825 INFO:tasks.rados.rados.0.smithi110.stdout:3476: read oid 23 snap 339 2024-09-14T03:49:32.825 INFO:tasks.rados.rados.0.smithi110.stdout:3476: expect (ObjNum 1065 snap 336 seq_num 1065) 2024-09-14T03:49:32.825 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 342 2024-09-14T03:49:32.825 INFO:tasks.rados.rados.0.smithi110.stdout:3477: seq_num 1080 ranges {3661824=491520,4153344=540672,4694016=262144} 2024-09-14T03:49:32.847 INFO:tasks.rados.rados.0.smithi110.stdout:3477: writing smithi11075562-1 from 3661824 to 4153344 tid 1 2024-09-14T03:49:32.850 INFO:tasks.rados.rados.0.smithi110.stdout:3477: writing smithi11075562-1 from 4153344 to 4694016 tid 2 2024-09-14T03:49:32.852 INFO:tasks.rados.rados.0.smithi110.stdout:3477: writing smithi11075562-1 from 4694016 to 4956160 tid 3 2024-09-14T03:49:32.852 INFO:tasks.rados.rados.0.smithi110.stdout:3478: delete oid 37 current snap is 342 2024-09-14T03:49:32.854 INFO:tasks.rados.rados.0.smithi110.stdout:3476: expect (ObjNum 1065 snap 336 seq_num 1065) 2024-09-14T03:49:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:32 smithi110 ceph-mon[88262]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T03:49:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:32 smithi110 ceph-mon[86423]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T03:49:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:32 smithi195 ceph-mon[82388]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T03:49:33.455 INFO:tasks.rados.rados.0.smithi110.stdout:3477: finishing write tid 1 to smithi11075562-1 2024-09-14T03:49:33.455 INFO:tasks.rados.rados.0.smithi110.stdout:3477: finishing write tid 2 to smithi11075562-1 2024-09-14T03:49:33.455 INFO:tasks.rados.rados.0.smithi110.stdout:3475: done (3 left) 2024-09-14T03:49:33.456 INFO:tasks.rados.rados.0.smithi110.stdout:3476: done (2 left) 2024-09-14T03:49:33.456 INFO:tasks.rados.rados.0.smithi110.stdout:3478: done (1 left) 2024-09-14T03:49:33.456 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 24 current snap is 342 2024-09-14T03:49:33.456 INFO:tasks.rados.rados.0.smithi110.stdout:3477: finishing write tid 3 to smithi11075562-1 2024-09-14T03:49:33.456 INFO:tasks.rados.rados.0.smithi110.stdout:3477: finishing write tid 4 to smithi11075562-1 2024-09-14T03:49:33.456 INFO:tasks.rados.rados.0.smithi110.stdout:3479: seq_num 1082 ranges {7544832=524288,8069120=704512,8773632=98304} 2024-09-14T03:49:33.497 INFO:tasks.rados.rados.0.smithi110.stdout:3479: writing smithi11075562-24 from 7544832 to 8069120 tid 1 2024-09-14T03:49:33.501 INFO:tasks.rados.rados.0.smithi110.stdout:3479: writing smithi11075562-24 from 8069120 to 8773632 tid 2 2024-09-14T03:49:33.501 INFO:tasks.rados.rados.0.smithi110.stdout:3479: writing smithi11075562-24 from 8773632 to 8871936 tid 3 2024-09-14T03:49:33.501 INFO:tasks.rados.rados.0.smithi110.stdout:3480: snap_create 2024-09-14T03:49:33.501 INFO:tasks.rados.rados.0.smithi110.stdout:3477: finishing write tid 5 to smithi11075562-1 2024-09-14T03:49:33.502 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2166 (ObjNum 1080 snap 342 seq_num 1080) dirty exists 2024-09-14T03:49:33.502 INFO:tasks.rados.rados.0.smithi110.stdout:3477: left oid 1 (ObjNum 1080 snap 342 seq_num 1080) 2024-09-14T03:49:33.504 INFO:tasks.rados.rados.0.smithi110.stdout:3479: finishing write tid 1 to smithi11075562-24 2024-09-14T03:49:33.505 INFO:tasks.rados.rados.0.smithi110.stdout:3479: finishing write tid 2 to smithi11075562-24 2024-09-14T03:49:33.506 INFO:tasks.rados.rados.0.smithi110.stdout:3479: finishing write tid 3 to smithi11075562-24 2024-09-14T03:49:33.506 INFO:tasks.rados.rados.0.smithi110.stdout:3479: finishing write tid 4 to smithi11075562-24 2024-09-14T03:49:33.507 INFO:tasks.rados.rados.0.smithi110.stdout:3479: finishing write tid 5 to smithi11075562-24 2024-09-14T03:49:33.507 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2028 (ObjNum 1082 snap 342 seq_num 1082) dirty exists 2024-09-14T03:49:33.507 INFO:tasks.rados.rados.0.smithi110.stdout:3479: left oid 24 (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:49:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3477: done (2 left) 2024-09-14T03:49:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3479: done (1 left) 2024-09-14T03:49:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3480: done (0 left) 2024-09-14T03:49:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3481: read oid 24 snap -1 2024-09-14T03:49:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3481: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:49:33.809 INFO:tasks.rados.rados.0.smithi110.stdout:3482: delete oid 50 current snap is 343 2024-09-14T03:49:33.810 INFO:tasks.rados.rados.0.smithi110.stdout:3482: done (1 left) 2024-09-14T03:49:33.810 INFO:tasks.rados.rados.0.smithi110.stdout:3483: rollback oid 26 current snap is 343 2024-09-14T03:49:33.810 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 26 to 342 2024-09-14T03:49:33.810 INFO:tasks.rados.rados.0.smithi110.stdout:3484: read oid 20 snap 341 2024-09-14T03:49:33.810 INFO:tasks.rados.rados.0.smithi110.stdout:3484: expect deleted 2024-09-14T03:49:33.810 INFO:tasks.rados.rados.0.smithi110.stdout:3485: snap_remove snap 339 2024-09-14T03:49:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:33 smithi110 ceph-mon[86423]: pgmap v777: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 856 KiB/s wr, 10 op/s 2024-09-14T03:49:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:33 smithi110 ceph-mon[86423]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T03:49:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:33 smithi110 ceph-mon[86423]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T03:49:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:33 smithi110 ceph-mon[88262]: pgmap v777: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 856 KiB/s wr, 10 op/s 2024-09-14T03:49:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:33 smithi110 ceph-mon[88262]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T03:49:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:33 smithi110 ceph-mon[88262]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T03:49:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:33 smithi195 ceph-mon[82388]: pgmap v777: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 856 KiB/s wr, 10 op/s 2024-09-14T03:49:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:33 smithi195 ceph-mon[82388]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T03:49:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:33 smithi195 ceph-mon[82388]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T03:49:34.812 INFO:tasks.rados.rados.0.smithi110.stdout:3485: done (3 left) 2024-09-14T03:49:34.812 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 343 2024-09-14T03:49:34.812 INFO:tasks.rados.rados.0.smithi110.stdout:3486: seq_num 1084 ranges {0=557056,557056=475136,1032192=327680} 2024-09-14T03:49:34.815 INFO:tasks.rados.rados.0.smithi110.stdout:3486: writing smithi11075562-16 from 0 to 557056 tid 1 2024-09-14T03:49:34.818 INFO:tasks.rados.rados.0.smithi110.stdout:3486: writing smithi11075562-16 from 557056 to 1032192 tid 2 2024-09-14T03:49:34.820 INFO:tasks.rados.rados.0.smithi110.stdout:3486: writing smithi11075562-16 from 1032192 to 1359872 tid 3 2024-09-14T03:49:34.820 INFO:tasks.rados.rados.0.smithi110.stdout:3487: copy_from oid 41 from oid 44 current snap is 343 2024-09-14T03:49:34.820 INFO:tasks.rados.rados.0.smithi110.stdout:3483: finishing rollback tid 0 to smithi11075562-26 2024-09-14T03:49:34.820 INFO:tasks.rados.rados.0.smithi110.stdout:3483: finishing rollback tid 1 to smithi11075562-26 2024-09-14T03:49:34.820 INFO:tasks.rados.rados.0.smithi110.stdout:3483: finishing rollback tid 2 to smithi11075562-26 2024-09-14T03:49:34.820 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 2000 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:49:34.821 INFO:tasks.rados.rados.0.smithi110.stdout:3481: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:49:35.534 INFO:tasks.rados.rados.0.smithi110.stdout:3486: finishing write tid 1 to smithi11075562-16 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3486: finishing write tid 2 to smithi11075562-16 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3486: finishing write tid 3 to smithi11075562-16 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3486: finishing write tid 4 to smithi11075562-16 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3486: finishing write tid 5 to smithi11075562-16 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2150 (ObjNum 1084 snap 343 seq_num 1084) dirty exists 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3486: left oid 16 (ObjNum 1084 snap 343 seq_num 1084) 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3487: finishing copy_from racing read to smithi11075562-41 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3487: finishing copy_from to smithi11075562-41 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 2143 (ObjNum 1008 snap 323 seq_num 1008) dirty exists 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3481: done (4 left) 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3483: done (3 left) 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3484: done (2 left) 2024-09-14T03:49:35.535 INFO:tasks.rados.rados.0.smithi110.stdout:3486: done (1 left) 2024-09-14T03:49:35.536 INFO:tasks.rados.rados.0.smithi110.stdout:3487: done (0 left) 2024-09-14T03:49:35.536 INFO:tasks.rados.rados.0.smithi110.stdout:3488: read oid 23 snap -1 2024-09-14T03:49:35.536 INFO:tasks.rados.rados.0.smithi110.stdout:3488: expect (ObjNum 1065 snap 336 seq_num 1065) 2024-09-14T03:49:35.536 INFO:tasks.rados.rados.0.smithi110.stdout:3489: snap_remove snap 341 2024-09-14T03:49:35.828 INFO:tasks.rados.rados.0.smithi110.stdout:3488: expect (ObjNum 1065 snap 336 seq_num 1065) 2024-09-14T03:49:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:35 smithi110 ceph-mon[88262]: pgmap v780: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 596 KiB/s wr, 10 op/s 2024-09-14T03:49:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:35 smithi110 ceph-mon[88262]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T03:49:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:35 smithi110 ceph-mon[86423]: pgmap v780: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 596 KiB/s wr, 10 op/s 2024-09-14T03:49:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:35 smithi110 ceph-mon[86423]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T03:49:36.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:35 smithi195 ceph-mon[82388]: pgmap v780: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 596 KiB/s wr, 10 op/s 2024-09-14T03:49:36.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:35 smithi195 ceph-mon[82388]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T03:49:36.431 INFO:tasks.rados.rados.0.smithi110.stdout:3488: done (1 left) 2024-09-14T03:49:36.432 INFO:tasks.rados.rados.0.smithi110.stdout:3489: done (0 left) 2024-09-14T03:49:36.432 INFO:tasks.rados.rados.0.smithi110.stdout:3490: snap_remove snap 316 2024-09-14T03:49:36.833 INFO:tasks.rados.rados.0.smithi110.stdout:3490: done (0 left) 2024-09-14T03:49:36.833 INFO:tasks.rados.rados.0.smithi110.stdout:3491: copy_from oid 22 from oid 26 current snap is 343 2024-09-14T03:49:36.833 INFO:tasks.rados.rados.0.smithi110.stdout:3492: read oid 48 snap -1 2024-09-14T03:49:36.833 INFO:tasks.rados.rados.0.smithi110.stdout:3492: expect (ObjNum 1045 snap 330 seq_num 1045) 2024-09-14T03:49:36.833 INFO:tasks.rados.rados.0.smithi110.stdout:3493: rmattr oid 19 current snap is 343 2024-09-14T03:49:36.833 INFO:tasks.rados.rados.0.smithi110.stdout:3493: done (2 left) 2024-09-14T03:49:36.833 INFO:tasks.rados.rados.0.smithi110.stdout:3494: delete oid 25 current snap is 343 2024-09-14T03:49:36.858 INFO:tasks.rados.rados.0.smithi110.stdout:3491: finishing copy_from racing read to smithi11075562-22 2024-09-14T03:49:36.860 INFO:tasks.rados.rados.0.smithi110.stdout:3494: done (2 left) 2024-09-14T03:49:36.860 INFO:tasks.rados.rados.0.smithi110.stdout:3495: delete oid 34 current snap is 343 2024-09-14T03:49:36.861 INFO:tasks.rados.rados.0.smithi110.stdout:3492: expect (ObjNum 1045 snap 330 seq_num 1045) 2024-09-14T03:49:36.953 INFO:tasks.rados.rados.0.smithi110.stdout:3491: finishing copy_from to smithi11075562-22 2024-09-14T03:49:36.953 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2000 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-14T03:49:36.953 INFO:tasks.rados.rados.0.smithi110.stdout:3491: done (2 left) 2024-09-14T03:49:36.953 INFO:tasks.rados.rados.0.smithi110.stdout:3492: done (1 left) 2024-09-14T03:49:36.953 INFO:tasks.rados.rados.0.smithi110.stdout:3495: done (0 left) 2024-09-14T03:49:36.953 INFO:tasks.rados.rados.0.smithi110.stdout:3496: delete oid 37 current snap is 343 2024-09-14T03:49:36.955 INFO:tasks.rados.rados.0.smithi110.stdout:3496: done (0 left) 2024-09-14T03:49:36.955 INFO:tasks.rados.rados.0.smithi110.stdout:3497: rmattr oid 49 current snap is 343 2024-09-14T03:49:36.955 INFO:tasks.rados.rados.0.smithi110.stdout:3497: done (0 left) 2024-09-14T03:49:36.955 INFO:tasks.rados.rados.0.smithi110.stdout:3498: snap_remove snap 342 2024-09-14T03:49:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:36 smithi110 ceph-mon[88262]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T03:49:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:36 smithi110 ceph-mon[86423]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T03:49:37.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:36 smithi195 ceph-mon[82388]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T03:49:37.209 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:49:36 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:49:36.852+0000 7f0e0734f640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 18b after trimming was started 2024-09-14T03:49:37.838 INFO:tasks.rados.rados.0.smithi110.stdout:3498: done (0 left) 2024-09-14T03:49:37.839 INFO:tasks.rados.rados.0.smithi110.stdout:3499: rmattr oid 50 current snap is 343 2024-09-14T03:49:37.839 INFO:tasks.rados.rados.0.smithi110.stdout:3499: done (0 left) 2024-09-14T03:49:37.839 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 343 2024-09-14T03:49:37.839 INFO:tasks.rados.rados.0.smithi110.stdout:3500: seq_num 1088 ranges {8249344=630784,8880128=466944,9347072=335872} 2024-09-14T03:49:37.885 INFO:tasks.rados.rados.0.smithi110.stdout:3500: writing smithi11075562-44 from 8249344 to 8880128 tid 1 2024-09-14T03:49:37.888 INFO:tasks.rados.rados.0.smithi110.stdout:3500: writing smithi11075562-44 from 8880128 to 9347072 tid 2 2024-09-14T03:49:37.891 INFO:tasks.rados.rados.0.smithi110.stdout:3500: writing smithi11075562-44 from 9347072 to 9682944 tid 3 2024-09-14T03:49:37.891 INFO:tasks.rados.rados.0.smithi110.stdout:3501: rollback oid 4 current snap is 343 2024-09-14T03:49:37.891 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 4 to 340 2024-09-14T03:49:37.891 INFO:tasks.rados.rados.0.smithi110.stdout:3502: read oid 2 snap -1 2024-09-14T03:49:37.891 INFO:tasks.rados.rados.0.smithi110.stdout:3502: expect (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:49:37.891 INFO:tasks.rados.rados.0.smithi110.stdout:3503: delete oid 40 current snap is 343 2024-09-14T03:49:37.891 INFO:tasks.rados.rados.0.smithi110.stdout:3500: finishing write tid 1 to smithi11075562-44 2024-09-14T03:49:37.895 INFO:tasks.rados.rados.0.smithi110.stdout:3500: finishing write tid 2 to smithi11075562-44 2024-09-14T03:49:37.896 INFO:tasks.rados.rados.0.smithi110.stdout:3502: expect (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:49:37.925 INFO:tasks.rados.rados.0.smithi110.stdout:3500: finishing write tid 3 to smithi11075562-44 2024-09-14T03:49:37.925 INFO:tasks.rados.rados.0.smithi110.stdout:3500: finishing write tid 4 to smithi11075562-44 2024-09-14T03:49:37.925 INFO:tasks.rados.rados.0.smithi110.stdout:3501: finishing rollback tid 0 to smithi11075562-4 2024-09-14T03:49:37.925 INFO:tasks.rados.rados.0.smithi110.stdout:3500: finishing write tid 5 to smithi11075562-44 2024-09-14T03:49:37.925 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 2147 (ObjNum 1088 snap 343 seq_num 1088) dirty exists 2024-09-14T03:49:37.925 INFO:tasks.rados.rados.0.smithi110.stdout:3500: left oid 44 (ObjNum 1088 snap 343 seq_num 1088) 2024-09-14T03:49:37.925 INFO:tasks.rados.rados.0.smithi110.stdout:3501: finishing rollback tid 1 to smithi11075562-4 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:3501: finishing rollback tid 2 to smithi11075562-4 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2157 (ObjNum 1069 snap 337 seq_num 1069) dirty exists 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:3500: done (3 left) 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:3501: done (2 left) 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:3502: done (1 left) 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:3503: done (0 left) 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:3504: rollback oid 45 current snap is 343 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 340 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 343 2024-09-14T03:49:37.926 INFO:tasks.rados.rados.0.smithi110.stdout:3505: seq_num 1090 ranges {18235392=663552,18898944=630784,19529728=172032} 2024-09-14T03:49:38.023 INFO:tasks.rados.rados.0.smithi110.stdout:3505: writing smithi11075562-39 from 18235392 to 18898944 tid 1 2024-09-14T03:49:38.026 INFO:tasks.rados.rados.0.smithi110.stdout:3505: writing smithi11075562-39 from 18898944 to 19529728 tid 2 2024-09-14T03:49:38.028 INFO:tasks.rados.rados.0.smithi110.stdout:3505: writing smithi11075562-39 from 19529728 to 19701760 tid 3 2024-09-14T03:49:38.028 INFO:tasks.rados.rados.0.smithi110.stdout:3506: copy_from oid 15 from oid 43 current snap is 343 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3504: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3504: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3504: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2462 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3504: done (2 left) 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3507: rmattr oid 29 current snap is 343 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3507: done (2 left) 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3508: rmattr oid 18 current snap is 343 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3509: read oid 31 snap -1 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3509: expect deleted 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3510: rollback oid 42 current snap is 343 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 42 to 340 2024-09-14T03:49:38.029 INFO:tasks.rados.rados.0.smithi110.stdout:3511: read oid 4 snap -1 2024-09-14T03:49:38.030 INFO:tasks.rados.rados.0.smithi110.stdout:3511: expect (ObjNum 1069 snap 337 seq_num 1069) 2024-09-14T03:49:38.030 INFO:tasks.rados.rados.0.smithi110.stdout:3512: read oid 45 snap -1 2024-09-14T03:49:38.030 INFO:tasks.rados.rados.0.smithi110.stdout:3512: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:49:38.030 INFO:tasks.rados.rados.0.smithi110.stdout:3513: setattr oid 33 current snap is 343 2024-09-14T03:49:38.030 INFO:tasks.rados.rados.0.smithi110.stdout:3505: finishing write tid 1 to smithi11075562-39 2024-09-14T03:49:38.030 INFO:tasks.rados.rados.0.smithi110.stdout:3509: done (7 left) 2024-09-14T03:49:38.031 INFO:tasks.rados.rados.0.smithi110.stdout:3514: delete oid 32 current snap is 343 2024-09-14T03:49:38.033 INFO:tasks.rados.rados.0.smithi110.stdout:3512: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-14T03:49:38.062 INFO:tasks.rados.rados.0.smithi110.stdout:3510: finishing rollback tid 0 to smithi11075562-42 2024-09-14T03:49:38.062 INFO:tasks.rados.rados.0.smithi110.stdout:3505: finishing write tid 2 to smithi11075562-39 2024-09-14T03:49:38.062 INFO:tasks.rados.rados.0.smithi110.stdout:3510: finishing rollback tid 1 to smithi11075562-42 2024-09-14T03:49:38.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 2152 (ObjNum 1055 snap 332 seq_num 1055) dirty exists 2024-09-14T03:49:38.062 INFO:tasks.rados.rados.0.smithi110.stdout:3505: finishing write tid 3 to smithi11075562-39 2024-09-14T03:49:38.062 INFO:tasks.rados.rados.0.smithi110.stdout:3510: finishing rollback tid 2 to smithi11075562-42 2024-09-14T03:49:38.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2184 (ObjNum 1038 snap 330 seq_num 1038) dirty exists 2024-09-14T03:49:38.062 INFO:tasks.rados.rados.0.smithi110.stdout:3505: finishing write tid 4 to smithi11075562-39 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3508: done (7 left) 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3510: done (6 left) 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3512: done (5 left) 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3514: done (4 left) 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3515: read oid 25 snap -1 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3515: expect deleted 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3516: copy_from oid 8 from oid 42 current snap is 343 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 2139 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3505: finishing write tid 5 to smithi11075562-39 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 2156 (ObjNum 1090 snap 343 seq_num 1090) dirty exists 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3505: left oid 39 (ObjNum 1090 snap 343 seq_num 1090) 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3506: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:49:38.063 INFO:tasks.rados.rados.0.smithi110.stdout:3506: finishing copy_from to smithi11075562-15 2024-09-14T03:49:38.064 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2151 (ObjNum 1055 snap 332 seq_num 1055) dirty exists 2024-09-14T03:49:38.064 INFO:tasks.rados.rados.0.smithi110.stdout:3505: done (5 left) 2024-09-14T03:49:38.064 INFO:tasks.rados.rados.0.smithi110.stdout:3506: done (4 left) 2024-09-14T03:49:38.064 INFO:tasks.rados.rados.0.smithi110.stdout:3513: done (3 left) 2024-09-14T03:49:38.064 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 47 current snap is 343 2024-09-14T03:49:38.064 INFO:tasks.rados.rados.0.smithi110.stdout:3517: seq_num 1092 ranges {0=696320,696320=802816} 2024-09-14T03:49:38.065 INFO:tasks.rados.rados.0.smithi110.stdout:3517: writing smithi11075562-47 from 0 to 696320 tid 1 2024-09-14T03:49:38.070 INFO:tasks.rados.rados.0.smithi110.stdout:3517: writing smithi11075562-47 from 696320 to 1499136 tid 2 2024-09-14T03:49:38.070 INFO:tasks.rados.rados.0.smithi110.stdout:3518: snap_remove snap 340 2024-09-14T03:49:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:37 smithi110 ceph-mon[88262]: pgmap v783: 121 pgs: 121 active+clean; 246 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 940 KiB/s wr, 15 op/s 2024-09-14T03:49:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:37 smithi110 ceph-mon[88262]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T03:49:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:37 smithi110 ceph-mon[86423]: pgmap v783: 121 pgs: 121 active+clean; 246 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 940 KiB/s wr, 15 op/s 2024-09-14T03:49:38.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:37 smithi110 ceph-mon[86423]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T03:49:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:37 smithi195 ceph-mon[82388]: pgmap v783: 121 pgs: 121 active+clean; 246 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 940 KiB/s wr, 15 op/s 2024-09-14T03:49:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:37 smithi195 ceph-mon[82388]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T03:49:38.810 INFO:tasks.rados.rados.0.smithi110.stdout:3511: expect (ObjNum 1069 snap 337 seq_num 1069) 2024-09-14T03:49:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:38 smithi110 ceph-mon[88262]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T03:49:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:38 smithi110 ceph-mon[88262]: pgmap v786: 121 pgs: 121 active+clean; 250 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.5 MiB/s wr, 16 op/s 2024-09-14T03:49:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:38 smithi110 ceph-mon[88262]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T03:49:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:38 smithi110 ceph-mon[86423]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T03:49:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:38 smithi110 ceph-mon[86423]: pgmap v786: 121 pgs: 121 active+clean; 250 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.5 MiB/s wr, 16 op/s 2024-09-14T03:49:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:38 smithi110 ceph-mon[86423]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T03:49:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:38 smithi195 ceph-mon[82388]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T03:49:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:38 smithi195 ceph-mon[82388]: pgmap v786: 121 pgs: 121 active+clean; 250 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.5 MiB/s wr, 16 op/s 2024-09-14T03:49:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:38 smithi195 ceph-mon[82388]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T03:49:39.561 INFO:tasks.rados.rados.0.smithi110.stdout:3517: finishing write tid 1 to smithi11075562-47 2024-09-14T03:49:39.561 INFO:tasks.rados.rados.0.smithi110.stdout:3517: finishing write tid 2 to smithi11075562-47 2024-09-14T03:49:39.561 INFO:tasks.rados.rados.0.smithi110.stdout:3517: finishing write tid 3 to smithi11075562-47 2024-09-14T03:49:39.561 INFO:tasks.rados.rados.0.smithi110.stdout:3516: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:49:39.561 INFO:tasks.rados.rados.0.smithi110.stdout:3517: finishing write tid 4 to smithi11075562-47 2024-09-14T03:49:39.561 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 2466 (ObjNum 1092 snap 343 seq_num 1092) dirty exists 2024-09-14T03:49:39.561 INFO:tasks.rados.rados.0.smithi110.stdout:3517: left oid 47 (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:49:39.561 INFO:tasks.rados.rados.0.smithi110.stdout:3516: finishing copy_from to smithi11075562-8 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 2184 (ObjNum 1038 snap 330 seq_num 1038) dirty exists 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:3511: done (4 left) 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:3515: done (3 left) 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:3516: done (2 left) 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:3517: done (1 left) 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:3518: done (0 left) 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:3519: read oid 3 snap -1 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:3519: expect (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 343 2024-09-14T03:49:39.562 INFO:tasks.rados.rados.0.smithi110.stdout:3520: seq_num 1093 ranges {10715136=712704,11427840=688128,12115968=114688} 2024-09-14T03:49:39.620 INFO:tasks.rados.rados.0.smithi110.stdout:3520: writing smithi11075562-5 from 10715136 to 11427840 tid 1 2024-09-14T03:49:39.623 INFO:tasks.rados.rados.0.smithi110.stdout:3520: writing smithi11075562-5 from 11427840 to 12115968 tid 2 2024-09-14T03:49:39.625 INFO:tasks.rados.rados.0.smithi110.stdout:3520: writing smithi11075562-5 from 12115968 to 12230656 tid 3 2024-09-14T03:49:39.625 INFO:tasks.rados.rados.0.smithi110.stdout:3521: read oid 44 snap -1 2024-09-14T03:49:39.625 INFO:tasks.rados.rados.0.smithi110.stdout:3521: expect (ObjNum 1088 snap 343 seq_num 1088) 2024-09-14T03:49:39.625 INFO:tasks.rados.rados.0.smithi110.stdout:3522: copy_from oid 30 from oid 47 current snap is 343 2024-09-14T03:49:39.625 INFO:tasks.rados.rados.0.smithi110.stdout:3519: expect (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:49:39.748 INFO:tasks.rados.rados.0.smithi110.stdout:3520: finishing write tid 1 to smithi11075562-5 2024-09-14T03:49:39.748 INFO:tasks.rados.rados.0.smithi110.stdout:3520: finishing write tid 2 to smithi11075562-5 2024-09-14T03:49:39.748 INFO:tasks.rados.rados.0.smithi110.stdout:3519: done (3 left) 2024-09-14T03:49:39.748 INFO:tasks.rados.rados.0.smithi110.stdout:3523: read oid 6 snap -1 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3523: expect (ObjNum 953 snap 307 seq_num 953) 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3524: snap_create 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3520: finishing write tid 3 to smithi11075562-5 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3520: finishing write tid 4 to smithi11075562-5 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3520: finishing write tid 5 to smithi11075562-5 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2476 (ObjNum 1093 snap 343 seq_num 1093) dirty exists 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3520: left oid 5 (ObjNum 1093 snap 343 seq_num 1093) 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3522: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3522: finishing copy_from to smithi11075562-30 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 2466 (ObjNum 1092 snap 343 seq_num 1092) dirty exists 2024-09-14T03:49:39.749 INFO:tasks.rados.rados.0.smithi110.stdout:3521: expect (ObjNum 1088 snap 343 seq_num 1088) 2024-09-14T03:49:40.498 INFO:tasks.rados.rados.0.smithi110.stdout:3523: expect (ObjNum 953 snap 307 seq_num 953) 2024-09-14T03:49:40.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:49:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:49:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:49:41.035 INFO:tasks.rados.rados.0.smithi110.stdout:3520: done (4 left) 2024-09-14T03:49:41.035 INFO:tasks.rados.rados.0.smithi110.stdout:3521: done (3 left) 2024-09-14T03:49:41.035 INFO:tasks.rados.rados.0.smithi110.stdout:3522: done (2 left) 2024-09-14T03:49:41.035 INFO:tasks.rados.rados.0.smithi110.stdout:3523: done (1 left) 2024-09-14T03:49:41.035 INFO:tasks.rados.rados.0.smithi110.stdout:3524: done (0 left) 2024-09-14T03:49:41.035 INFO:tasks.rados.rados.0.smithi110.stdout:3525: snap_remove snap 343 2024-09-14T03:49:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:40 smithi110 ceph-mon[86423]: pgmap v788: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.6 MiB/s wr, 9 op/s 2024-09-14T03:49:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:40 smithi110 ceph-mon[86423]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T03:49:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:40 smithi110 ceph-mon[88262]: pgmap v788: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.6 MiB/s wr, 9 op/s 2024-09-14T03:49:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:40 smithi110 ceph-mon[88262]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T03:49:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:40 smithi195 ceph-mon[82388]: pgmap v788: 121 pgs: 121 active+clean; 203 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.6 MiB/s wr, 9 op/s 2024-09-14T03:49:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:40 smithi195 ceph-mon[82388]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T03:49:41.851 INFO:tasks.rados.rados.0.smithi110.stdout:3525: done (0 left) 2024-09-14T03:49:41.851 INFO:tasks.rados.rados.0.smithi110.stdout:3526: rollback oid 35 current snap is 344 2024-09-14T03:49:41.851 INFO:tasks.rados.rados.0.smithi110.stdout:3526: done (0 left) 2024-09-14T03:49:41.851 INFO:tasks.rados.rados.0.smithi110.stdout:3527: snap_create 2024-09-14T03:49:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:49:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:49:41] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-14T03:49:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3527: done (0 left) 2024-09-14T03:49:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 345 2024-09-14T03:49:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3528: seq_num 1094 ranges {0=729088,729088=573440,1302528=229376} 2024-09-14T03:49:42.857 INFO:tasks.rados.rados.0.smithi110.stdout:3528: writing smithi11075562-32 from 0 to 729088 tid 1 2024-09-14T03:49:42.861 INFO:tasks.rados.rados.0.smithi110.stdout:3528: writing smithi11075562-32 from 729088 to 1302528 tid 2 2024-09-14T03:49:42.863 INFO:tasks.rados.rados.0.smithi110.stdout:3528: writing smithi11075562-32 from 1302528 to 1531904 tid 3 2024-09-14T03:49:42.863 INFO:tasks.rados.rados.0.smithi110.stdout:3529: read oid 50 snap 344 2024-09-14T03:49:42.863 INFO:tasks.rados.rados.0.smithi110.stdout:3529: expect deleted 2024-09-14T03:49:42.863 INFO:tasks.rados.rados.0.smithi110.stdout:3530: copy_from oid 45 from oid 16 current snap is 345 2024-09-14T03:49:42.863 INFO:tasks.rados.rados.0.smithi110.stdout:3531: snap_create 2024-09-14T03:49:42.864 INFO:tasks.rados.rados.0.smithi110.stdout:3528: finishing write tid 1 to smithi11075562-32 2024-09-14T03:49:42.867 INFO:tasks.rados.rados.0.smithi110.stdout:3528: finishing write tid 2 to smithi11075562-32 2024-09-14T03:49:42.867 INFO:tasks.rados.rados.0.smithi110.stdout:3528: finishing write tid 3 to smithi11075562-32 2024-09-14T03:49:42.867 INFO:tasks.rados.rados.0.smithi110.stdout:3528: finishing write tid 4 to smithi11075562-32 2024-09-14T03:49:42.868 INFO:tasks.rados.rados.0.smithi110.stdout:3528: finishing write tid 5 to smithi11075562-32 2024-09-14T03:49:42.869 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2470 (ObjNum 1094 snap 345 seq_num 1094) dirty exists 2024-09-14T03:49:42.869 INFO:tasks.rados.rados.0.smithi110.stdout:3528: left oid 32 (ObjNum 1094 snap 345 seq_num 1094) 2024-09-14T03:49:42.874 INFO:tasks.rados.rados.0.smithi110.stdout:3530: finishing copy_from racing read to smithi11075562-45 2024-09-14T03:49:42.875 INFO:tasks.rados.rados.0.smithi110.stdout:3530: finishing copy_from to smithi11075562-45 2024-09-14T03:49:42.876 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2150 (ObjNum 1084 snap 343 seq_num 1084) dirty exists 2024-09-14T03:49:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:42 smithi110 ceph-mon[88262]: pgmap v790: 121 pgs: 121 active+clean; 156 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.8 MiB/s wr, 12 op/s 2024-09-14T03:49:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:42 smithi110 ceph-mon[88262]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T03:49:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:42 smithi110 ceph-mon[86423]: pgmap v790: 121 pgs: 121 active+clean; 156 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.8 MiB/s wr, 12 op/s 2024-09-14T03:49:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:42 smithi110 ceph-mon[86423]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T03:49:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:43.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:42 smithi195 ceph-mon[82388]: pgmap v790: 121 pgs: 121 active+clean; 156 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.8 MiB/s wr, 12 op/s 2024-09-14T03:49:43.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:42 smithi195 ceph-mon[82388]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T03:49:43.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:3528: done (3 left) 2024-09-14T03:49:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:3529: done (2 left) 2024-09-14T03:49:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:3530: done (1 left) 2024-09-14T03:49:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:3531: done (0 left) 2024-09-14T03:49:43.813 INFO:tasks.rados.rados.0.smithi110.stdout:3532: read oid 34 snap -1 2024-09-14T03:49:43.814 INFO:tasks.rados.rados.0.smithi110.stdout:3532: expect deleted 2024-09-14T03:49:43.814 INFO:tasks.rados.rados.0.smithi110.stdout:3533: copy_from oid 11 from oid 38 current snap is 346 2024-09-14T03:49:43.814 INFO:tasks.rados.rados.0.smithi110.stdout:3534: snap_create 2024-09-14T03:49:43.829 INFO:tasks.rados.rados.0.smithi110.stdout:3533: finishing copy_from racing read to smithi11075562-11 2024-09-14T03:49:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:3533: finishing copy_from to smithi11075562-11 2024-09-14T03:49:43.830 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 1632 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:49:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:43 smithi110 ceph-mon[88262]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T03:49:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:43 smithi110 ceph-mon[88262]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T03:49:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:43 smithi110 ceph-mon[86423]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T03:49:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:43 smithi110 ceph-mon[86423]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T03:49:44.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:43 smithi195 ceph-mon[82388]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T03:49:44.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:43 smithi195 ceph-mon[82388]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T03:49:44.817 INFO:tasks.rados.rados.0.smithi110.stdout:3532: done (2 left) 2024-09-14T03:49:44.818 INFO:tasks.rados.rados.0.smithi110.stdout:3533: done (1 left) 2024-09-14T03:49:44.818 INFO:tasks.rados.rados.0.smithi110.stdout:3534: done (0 left) 2024-09-14T03:49:44.818 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 347 2024-09-14T03:49:44.818 INFO:tasks.rados.rados.0.smithi110.stdout:3535: seq_num 1095 ranges {1785856=745472,2531328=499712,3031040=303104} 2024-09-14T03:49:44.831 INFO:tasks.rados.rados.0.smithi110.stdout:3535: writing smithi11075562-26 from 1785856 to 2531328 tid 1 2024-09-14T03:49:44.834 INFO:tasks.rados.rados.0.smithi110.stdout:3535: writing smithi11075562-26 from 2531328 to 3031040 tid 2 2024-09-14T03:49:44.835 INFO:tasks.rados.rados.0.smithi110.stdout:3535: writing smithi11075562-26 from 3031040 to 3334144 tid 3 2024-09-14T03:49:44.836 INFO:tasks.rados.rados.0.smithi110.stdout:3536: rollback oid 10 current snap is 347 2024-09-14T03:49:44.836 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 344 2024-09-14T03:49:44.836 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 347 2024-09-14T03:49:44.836 INFO:tasks.rados.rados.0.smithi110.stdout:3537: seq_num 1096 ranges {1064960=761856,1826816=786432,2613248=16384} 2024-09-14T03:49:44.846 INFO:tasks.rados.rados.0.smithi110.stdout:3537: writing smithi11075562-49 from 1064960 to 1826816 tid 1 2024-09-14T03:49:44.852 INFO:tasks.rados.rados.0.smithi110.stdout:3537: writing smithi11075562-49 from 1826816 to 2613248 tid 2 2024-09-14T03:49:44.852 INFO:tasks.rados.rados.0.smithi110.stdout:3537: writing smithi11075562-49 from 2613248 to 2629632 tid 3 2024-09-14T03:49:44.852 INFO:tasks.rados.rados.0.smithi110.stdout:3538: read oid 25 snap -1 2024-09-14T03:49:44.852 INFO:tasks.rados.rados.0.smithi110.stdout:3538: expect deleted 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3539: read oid 24 snap -1 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3539: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3540: copy_from oid 12 from oid 9 current snap is 347 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3536: finishing rollback tid 0 to smithi11075562-10 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3535: finishing write tid 1 to smithi11075562-26 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3535: finishing write tid 2 to smithi11075562-26 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3536: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3535: finishing write tid 3 to smithi11075562-26 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3536: finishing rollback tid 2 to smithi11075562-10 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2031 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3535: finishing write tid 4 to smithi11075562-26 2024-09-14T03:49:44.853 INFO:tasks.rados.rados.0.smithi110.stdout:3535: finishing write tid 5 to smithi11075562-26 2024-09-14T03:49:44.854 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 2004 (ObjNum 1095 snap 347 seq_num 1095) dirty exists 2024-09-14T03:49:44.854 INFO:tasks.rados.rados.0.smithi110.stdout:3535: left oid 26 (ObjNum 1095 snap 347 seq_num 1095) 2024-09-14T03:49:44.854 INFO:tasks.rados.rados.0.smithi110.stdout:3537: finishing write tid 1 to smithi11075562-49 2024-09-14T03:49:44.854 INFO:tasks.rados.rados.0.smithi110.stdout:3535: done (5 left) 2024-09-14T03:49:44.854 INFO:tasks.rados.rados.0.smithi110.stdout:3536: done (4 left) 2024-09-14T03:49:44.854 INFO:tasks.rados.rados.0.smithi110.stdout:3541: read oid 7 snap -1 2024-09-14T03:49:44.854 INFO:tasks.rados.rados.0.smithi110.stdout:3541: expect (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:49:44.854 INFO:tasks.rados.rados.0.smithi110.stdout:3542: snap_create 2024-09-14T03:49:44.856 INFO:tasks.rados.rados.0.smithi110.stdout:3541: expect (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:49:44.892 INFO:tasks.rados.rados.0.smithi110.stdout:3537: finishing write tid 2 to smithi11075562-49 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:3537: finishing write tid 3 to smithi11075562-49 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:3537: finishing write tid 4 to smithi11075562-49 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:3537: finishing write tid 5 to smithi11075562-49 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2193 (ObjNum 1096 snap 347 seq_num 1096) dirty exists 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:3537: left oid 49 (ObjNum 1096 snap 347 seq_num 1096) 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:3540: finishing copy_from racing read to smithi11075562-12 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:3540: finishing copy_from to smithi11075562-12 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2137 (ObjNum 1063 snap 336 seq_num 1063) dirty exists 2024-09-14T03:49:44.893 INFO:tasks.rados.rados.0.smithi110.stdout:3539: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:49:45.821 INFO:tasks.rados.rados.0.smithi110.stdout:3537: done (5 left) 2024-09-14T03:49:45.821 INFO:tasks.rados.rados.0.smithi110.stdout:3538: done (4 left) 2024-09-14T03:49:45.821 INFO:tasks.rados.rados.0.smithi110.stdout:3539: done (3 left) 2024-09-14T03:49:45.821 INFO:tasks.rados.rados.0.smithi110.stdout:3540: done (2 left) 2024-09-14T03:49:45.821 INFO:tasks.rados.rados.0.smithi110.stdout:3541: done (1 left) 2024-09-14T03:49:45.821 INFO:tasks.rados.rados.0.smithi110.stdout:3542: done (0 left) 2024-09-14T03:49:45.821 INFO:tasks.rados.rados.0.smithi110.stdout:3543: rollback oid 20 current snap is 348 2024-09-14T03:49:45.822 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 346 2024-09-14T03:49:45.822 INFO:tasks.rados.rados.0.smithi110.stdout:3544: rmattr oid 23 current snap is 348 2024-09-14T03:49:45.822 INFO:tasks.rados.rados.0.smithi110.stdout:3544: done (1 left) 2024-09-14T03:49:45.822 INFO:tasks.rados.rados.0.smithi110.stdout:3545: rollback oid 31 current snap is 348 2024-09-14T03:49:45.822 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 345 2024-09-14T03:49:45.822 INFO:tasks.rados.rados.0.smithi110.stdout:3546: delete oid 39 current snap is 348 2024-09-14T03:49:45.822 INFO:tasks.rados.rados.0.smithi110.stdout:3545: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:49:45.822 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 0 (ObjNum 33 snap 0 seq_num 0) dirty dne 2024-09-14T03:49:45.823 INFO:tasks.rados.rados.0.smithi110.stdout:3543: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:49:45.823 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 0 (ObjNum 33 snap 0 seq_num 65151760) dirty dne 2024-09-14T03:49:45.824 INFO:tasks.rados.rados.0.smithi110.stdout:3543: done (2 left) 2024-09-14T03:49:45.824 INFO:tasks.rados.rados.0.smithi110.stdout:3545: done (1 left) 2024-09-14T03:49:45.825 INFO:tasks.rados.rados.0.smithi110.stdout:3546: done (0 left) 2024-09-14T03:49:45.825 INFO:tasks.rados.rados.0.smithi110.stdout:3547: delete oid 22 current snap is 348 2024-09-14T03:49:45.827 INFO:tasks.rados.rados.0.smithi110.stdout:3547: done (0 left) 2024-09-14T03:49:45.827 INFO:tasks.rados.rados.0.smithi110.stdout:3548: snap_remove snap 346 2024-09-14T03:49:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:45 smithi110 ceph-mon[88262]: pgmap v794: 121 pgs: 121 active+clean; 159 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-09-14T03:49:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:45 smithi110 ceph-mon[88262]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T03:49:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:45 smithi110 ceph-mon[86423]: pgmap v794: 121 pgs: 121 active+clean; 159 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-09-14T03:49:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:45 smithi110 ceph-mon[86423]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T03:49:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:45 smithi195 ceph-mon[82388]: pgmap v794: 121 pgs: 121 active+clean; 159 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-09-14T03:49:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:45 smithi195 ceph-mon[82388]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T03:49:46.826 INFO:tasks.rados.rados.0.smithi110.stdout:3548: done (0 left) 2024-09-14T03:49:46.827 INFO:tasks.rados.rados.0.smithi110.stdout:3549: delete oid 17 current snap is 348 2024-09-14T03:49:46.833 INFO:tasks.rados.rados.0.smithi110.stdout:3549: done (0 left) 2024-09-14T03:49:46.833 INFO:tasks.rados.rados.0.smithi110.stdout:3550: snap_create 2024-09-14T03:49:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:46 smithi110 ceph-mon[88262]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T03:49:47.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:46 smithi110 ceph-mon[86423]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T03:49:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:46 smithi195 ceph-mon[82388]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T03:49:47.833 INFO:tasks.rados.rados.0.smithi110.stdout:3550: done (0 left) 2024-09-14T03:49:47.833 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 349 2024-09-14T03:49:47.833 INFO:tasks.rados.rados.0.smithi110.stdout:3551: seq_num 1100 ranges {3817472=425984,4243456=729088,4972544=475136} 2024-09-14T03:49:47.855 INFO:tasks.rados.rados.0.smithi110.stdout:3551: writing smithi11075562-8 from 3817472 to 4243456 tid 1 2024-09-14T03:49:47.859 INFO:tasks.rados.rados.0.smithi110.stdout:3551: writing smithi11075562-8 from 4243456 to 4972544 tid 2 2024-09-14T03:49:47.862 INFO:tasks.rados.rados.0.smithi110.stdout:3551: writing smithi11075562-8 from 4972544 to 5447680 tid 3 2024-09-14T03:49:47.862 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 349 2024-09-14T03:49:47.862 INFO:tasks.rados.rados.0.smithi110.stdout:3552: seq_num 1101 ranges {0=442368,442368=606208,1048576=598016} 2024-09-14T03:49:47.864 INFO:tasks.rados.rados.0.smithi110.stdout:3552: writing smithi11075562-29 from 0 to 442368 tid 1 2024-09-14T03:49:47.868 INFO:tasks.rados.rados.0.smithi110.stdout:3552: writing smithi11075562-29 from 442368 to 1048576 tid 2 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3552: writing smithi11075562-29 from 1048576 to 1646592 tid 3 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3553: copy_from oid 12 from oid 17 current snap is 349 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3551: finishing write tid 1 to smithi11075562-8 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3551: finishing write tid 2 to smithi11075562-8 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3551: finishing write tid 3 to smithi11075562-8 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3551: finishing write tid 4 to smithi11075562-8 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3551: finishing write tid 5 to smithi11075562-8 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 2475 (ObjNum 1100 snap 349 seq_num 1100) dirty exists 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3551: left oid 8 (ObjNum 1100 snap 349 seq_num 1100) 2024-09-14T03:49:47.873 INFO:tasks.rados.rados.0.smithi110.stdout:3552: finishing write tid 1 to smithi11075562-29 2024-09-14T03:49:47.874 INFO:tasks.rados.rados.0.smithi110.stdout:3552: finishing write tid 2 to smithi11075562-29 2024-09-14T03:49:47.874 INFO:tasks.rados.rados.0.smithi110.stdout:3551: done (2 left) 2024-09-14T03:49:47.874 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 349 2024-09-14T03:49:47.874 INFO:tasks.rados.rados.0.smithi110.stdout:3554: seq_num 1102 ranges {0=458752,458752=491520,950272=491520,1441792=221184} 2024-09-14T03:49:47.875 INFO:tasks.rados.rados.0.smithi110.stdout:3554: writing smithi11075562-22 from 0 to 458752 tid 1 2024-09-14T03:49:47.878 INFO:tasks.rados.rados.0.smithi110.stdout:3554: writing smithi11075562-22 from 458752 to 950272 tid 2 2024-09-14T03:49:47.881 INFO:tasks.rados.rados.0.smithi110.stdout:3554: writing smithi11075562-22 from 950272 to 1441792 tid 3 2024-09-14T03:49:47.882 INFO:tasks.rados.rados.0.smithi110.stdout:3554: writing smithi11075562-22 from 1441792 to 1662976 tid 4 2024-09-14T03:49:47.882 INFO:tasks.rados.rados.0.smithi110.stdout:3555: snap_remove snap 344 2024-09-14T03:49:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:47 smithi110 ceph-mon[88262]: pgmap v797: 121 pgs: 121 active+clean; 168 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T03:49:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:47 smithi110 ceph-mon[88262]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T03:49:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:47 smithi110 ceph-mon[86423]: pgmap v797: 121 pgs: 121 active+clean; 168 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T03:49:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:47 smithi110 ceph-mon[86423]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T03:49:48.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:47 smithi195 ceph-mon[82388]: pgmap v797: 121 pgs: 121 active+clean; 168 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T03:49:48.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:47 smithi195 ceph-mon[82388]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T03:49:48.813 INFO:tasks.rados.rados.0.smithi110.stdout:3555: done (3 left) 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3556: rollback oid 40 current snap is 349 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 40 to 347 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3552: finishing write tid 3 to smithi11075562-29 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3552: finishing write tid 4 to smithi11075562-29 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3552: finishing write tid 5 to smithi11075562-29 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1640 (ObjNum 1101 snap 349 seq_num 1101) dirty exists 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3552: left oid 29 (ObjNum 1101 snap 349 seq_num 1101) 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3553: finishing copy_from to smithi11075562-12 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3553: got expected ENOENT (src dne) 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3553: finishing copy_from racing read to smithi11075562-12 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3554: finishing write tid 1 to smithi11075562-22 2024-09-14T03:49:48.814 INFO:tasks.rados.rados.0.smithi110.stdout:3554: finishing write tid 2 to smithi11075562-22 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3554: finishing write tid 3 to smithi11075562-22 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3554: finishing write tid 4 to smithi11075562-22 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3554: finishing write tid 5 to smithi11075562-22 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3554: finishing write tid 6 to smithi11075562-22 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2480 (ObjNum 1102 snap 349 seq_num 1102) dirty exists 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3554: left oid 22 (ObjNum 1102 snap 349 seq_num 1102) 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3552: done (3 left) 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3553: done (2 left) 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3554: done (1 left) 2024-09-14T03:49:48.815 INFO:tasks.rados.rados.0.smithi110.stdout:3557: snap_remove snap 348 2024-09-14T03:49:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:48 smithi110 ceph-mon[88262]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T03:49:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:48 smithi110 ceph-mon[88262]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T03:49:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:48 smithi110 ceph-mon[86423]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T03:49:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:48 smithi110 ceph-mon[86423]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T03:49:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:48 smithi195 ceph-mon[82388]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T03:49:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:48 smithi195 ceph-mon[82388]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T03:49:49.815 INFO:tasks.rados.rados.0.smithi110.stdout:3557: done (1 left) 2024-09-14T03:49:49.815 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 349 2024-09-14T03:49:49.815 INFO:tasks.rados.rados.0.smithi110.stdout:3558: seq_num 1103 ranges {3268608=475136,3743744=425984,4169728=696320,4866048=81920} 2024-09-14T03:49:49.836 INFO:tasks.rados.rados.0.smithi110.stdout:3558: writing smithi11075562-9 from 3268608 to 3743744 tid 1 2024-09-14T03:49:49.838 INFO:tasks.rados.rados.0.smithi110.stdout:3558: writing smithi11075562-9 from 3743744 to 4169728 tid 2 2024-09-14T03:49:49.842 INFO:tasks.rados.rados.0.smithi110.stdout:3558: writing smithi11075562-9 from 4169728 to 4866048 tid 3 2024-09-14T03:49:49.843 INFO:tasks.rados.rados.0.smithi110.stdout:3558: writing smithi11075562-9 from 4866048 to 4947968 tid 4 2024-09-14T03:49:49.843 INFO:tasks.rados.rados.0.smithi110.stdout:3559: snap_create 2024-09-14T03:49:49.843 INFO:tasks.rados.rados.0.smithi110.stdout:3556: finishing rollback tid 1 to smithi11075562-40 2024-09-14T03:49:49.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:49:49.844 INFO:tasks.rados.rados.0.smithi110.stdout:3558: finishing write tid 1 to smithi11075562-9 2024-09-14T03:49:49.844 INFO:tasks.rados.rados.0.smithi110.stdout:3558: finishing write tid 2 to smithi11075562-9 2024-09-14T03:49:49.847 INFO:tasks.rados.rados.0.smithi110.stdout:3558: finishing write tid 3 to smithi11075562-9 2024-09-14T03:49:49.848 INFO:tasks.rados.rados.0.smithi110.stdout:3558: finishing write tid 4 to smithi11075562-9 2024-09-14T03:49:49.848 INFO:tasks.rados.rados.0.smithi110.stdout:3558: finishing write tid 5 to smithi11075562-9 2024-09-14T03:49:49.849 INFO:tasks.rados.rados.0.smithi110.stdout:3558: finishing write tid 6 to smithi11075562-9 2024-09-14T03:49:49.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 2144 (ObjNum 1103 snap 349 seq_num 1103) dirty exists 2024-09-14T03:49:49.849 INFO:tasks.rados.rados.0.smithi110.stdout:3558: left oid 9 (ObjNum 1103 snap 349 seq_num 1103) 2024-09-14T03:49:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:49 smithi110 ceph-mon[88262]: pgmap v800: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2024-09-14T03:49:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:49 smithi110 ceph-mon[88262]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T03:49:50.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:49 smithi110 ceph-mon[86423]: pgmap v800: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2024-09-14T03:49:50.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:49 smithi110 ceph-mon[86423]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T03:49:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:49 smithi195 ceph-mon[82388]: pgmap v800: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2024-09-14T03:49:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:49 smithi195 ceph-mon[82388]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T03:49:50.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:49:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:49:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:49:50.844 INFO:tasks.rados.rados.0.smithi110.stdout:3556: done (2 left) 2024-09-14T03:49:50.844 INFO:tasks.rados.rados.0.smithi110.stdout:3558: done (1 left) 2024-09-14T03:49:50.844 INFO:tasks.rados.rados.0.smithi110.stdout:3559: done (0 left) 2024-09-14T03:49:50.845 INFO:tasks.rados.rados.0.smithi110.stdout:3560: read oid 31 snap -1 2024-09-14T03:49:50.845 INFO:tasks.rados.rados.0.smithi110.stdout:3560: expect deleted 2024-09-14T03:49:50.845 INFO:tasks.rados.rados.0.smithi110.stdout:3561: read oid 45 snap -1 2024-09-14T03:49:50.845 INFO:tasks.rados.rados.0.smithi110.stdout:3561: expect (ObjNum 1084 snap 343 seq_num 1084) 2024-09-14T03:49:50.845 INFO:tasks.rados.rados.0.smithi110.stdout:3562: rollback oid 32 current snap is 350 2024-09-14T03:49:50.845 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 349 2024-09-14T03:49:50.845 INFO:tasks.rados.rados.0.smithi110.stdout:3563: delete oid 48 current snap is 350 2024-09-14T03:49:50.848 INFO:tasks.rados.rados.0.smithi110.stdout:3562: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:49:50.849 INFO:tasks.rados.rados.0.smithi110.stdout:3562: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:49:50.850 INFO:tasks.rados.rados.0.smithi110.stdout:3562: finishing rollback tid 2 to smithi11075562-32 2024-09-14T03:49:50.850 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2483 (ObjNum 1094 snap 345 seq_num 1094) dirty exists 2024-09-14T03:49:50.850 INFO:tasks.rados.rados.0.smithi110.stdout:3560: done (3 left) 2024-09-14T03:49:50.850 INFO:tasks.rados.rados.0.smithi110.stdout:3562: done (2 left) 2024-09-14T03:49:50.851 INFO:tasks.rados.rados.0.smithi110.stdout:3563: done (1 left) 2024-09-14T03:49:50.851 INFO:tasks.rados.rados.0.smithi110.stdout:3564: rollback oid 30 current snap is 350 2024-09-14T03:49:50.851 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 349 2024-09-14T03:49:50.851 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 350 2024-09-14T03:49:50.851 INFO:tasks.rados.rados.0.smithi110.stdout:3565: seq_num 1105 ranges {2179072=516096,2695168=598016,3293184=606208} 2024-09-14T03:49:50.865 INFO:tasks.rados.rados.0.smithi110.stdout:3565: writing smithi11075562-19 from 2179072 to 2695168 tid 1 2024-09-14T03:49:50.868 INFO:tasks.rados.rados.0.smithi110.stdout:3565: writing smithi11075562-19 from 2695168 to 3293184 tid 2 2024-09-14T03:49:50.872 INFO:tasks.rados.rados.0.smithi110.stdout:3565: writing smithi11075562-19 from 3293184 to 3899392 tid 3 2024-09-14T03:49:50.873 INFO:tasks.rados.rados.0.smithi110.stdout:3566: snap_create 2024-09-14T03:49:50.873 INFO:tasks.rados.rados.0.smithi110.stdout:3564: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:49:50.873 INFO:tasks.rados.rados.0.smithi110.stdout:3564: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:49:50.873 INFO:tasks.rados.rados.0.smithi110.stdout:3564: finishing rollback tid 2 to smithi11075562-30 2024-09-14T03:49:50.873 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 2486 (ObjNum 1092 snap 343 seq_num 1092) dirty exists 2024-09-14T03:49:50.873 INFO:tasks.rados.rados.0.smithi110.stdout:3561: expect (ObjNum 1084 snap 343 seq_num 1084) 2024-09-14T03:49:50.909 INFO:tasks.rados.rados.0.smithi110.stdout:3565: finishing write tid 1 to smithi11075562-19 2024-09-14T03:49:50.909 INFO:tasks.rados.rados.0.smithi110.stdout:3565: finishing write tid 2 to smithi11075562-19 2024-09-14T03:49:50.909 INFO:tasks.rados.rados.0.smithi110.stdout:3565: finishing write tid 3 to smithi11075562-19 2024-09-14T03:49:50.910 INFO:tasks.rados.rados.0.smithi110.stdout:3565: finishing write tid 4 to smithi11075562-19 2024-09-14T03:49:50.910 INFO:tasks.rados.rados.0.smithi110.stdout:3565: finishing write tid 5 to smithi11075562-19 2024-09-14T03:49:50.910 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2156 (ObjNum 1105 snap 350 seq_num 1105) dirty exists 2024-09-14T03:49:50.910 INFO:tasks.rados.rados.0.smithi110.stdout:3565: left oid 19 (ObjNum 1105 snap 350 seq_num 1105) 2024-09-14T03:49:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:50 smithi110 ceph-mon[86423]: pgmap v803: 121 pgs: 121 active+clean; 173 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-14T03:49:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:50 smithi110 ceph-mon[88262]: pgmap v803: 121 pgs: 121 active+clean; 173 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-14T03:49:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:50 smithi195 ceph-mon[82388]: pgmap v803: 121 pgs: 121 active+clean; 173 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-14T03:49:51.840 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:49:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:49:51] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.51.0" 2024-09-14T03:49:51.849 INFO:tasks.rados.rados.0.smithi110.stdout:3561: done (3 left) 2024-09-14T03:49:51.849 INFO:tasks.rados.rados.0.smithi110.stdout:3564: done (2 left) 2024-09-14T03:49:51.849 INFO:tasks.rados.rados.0.smithi110.stdout:3565: done (1 left) 2024-09-14T03:49:51.849 INFO:tasks.rados.rados.0.smithi110.stdout:3566: done (0 left) 2024-09-14T03:49:51.849 INFO:tasks.rados.rados.0.smithi110.stdout:3567: rollback oid 16 current snap is 351 2024-09-14T03:49:51.849 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 349 2024-09-14T03:49:51.849 INFO:tasks.rados.rados.0.smithi110.stdout:3568: setattr oid 43 current snap is 351 2024-09-14T03:49:51.849 INFO:tasks.rados.rados.0.smithi110.stdout:3569: delete oid 40 current snap is 351 2024-09-14T03:49:51.854 INFO:tasks.rados.rados.0.smithi110.stdout:3567: finishing rollback tid 0 to smithi11075562-16 2024-09-14T03:49:51.854 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 2158 (ObjNum 1055 snap 332 seq_num 1055) dirty exists 2024-09-14T03:49:51.855 INFO:tasks.rados.rados.0.smithi110.stdout:3567: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:49:51.855 INFO:tasks.rados.rados.0.smithi110.stdout:3567: finishing rollback tid 2 to smithi11075562-16 2024-09-14T03:49:51.855 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2159 (ObjNum 1084 snap 343 seq_num 1084) dirty exists 2024-09-14T03:49:51.856 INFO:tasks.rados.rados.0.smithi110.stdout:3567: done (2 left) 2024-09-14T03:49:51.856 INFO:tasks.rados.rados.0.smithi110.stdout:3568: done (1 left) 2024-09-14T03:49:51.856 INFO:tasks.rados.rados.0.smithi110.stdout:3569: done (0 left) 2024-09-14T03:49:51.856 INFO:tasks.rados.rados.0.smithi110.stdout:3570: copy_from oid 13 from oid 10 current snap is 351 2024-09-14T03:49:51.856 INFO:tasks.rados.rados.0.smithi110.stdout:3571: setattr oid 22 current snap is 351 2024-09-14T03:49:51.857 INFO:tasks.rados.rados.0.smithi110.stdout:3572: copy_from oid 50 from oid 32 current snap is 351 2024-09-14T03:49:51.858 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 351 2024-09-14T03:49:51.858 INFO:tasks.rados.rados.0.smithi110.stdout:3573: seq_num 1107 ranges {9363456=548864,9912320=770048,10682368=434176} 2024-09-14T03:49:51.912 INFO:tasks.rados.rados.0.smithi110.stdout:3573: writing smithi11075562-4 from 9363456 to 9912320 tid 1 2024-09-14T03:49:51.916 INFO:tasks.rados.rados.0.smithi110.stdout:3573: writing smithi11075562-4 from 9912320 to 10682368 tid 2 2024-09-14T03:49:51.919 INFO:tasks.rados.rados.0.smithi110.stdout:3573: writing smithi11075562-4 from 10682368 to 11116544 tid 3 2024-09-14T03:49:51.919 INFO:tasks.rados.rados.0.smithi110.stdout:3574: snap_create 2024-09-14T03:49:51.920 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2487 (ObjNum 1102 snap 349 seq_num 1102) dirty exists 2024-09-14T03:49:51.920 INFO:tasks.rados.rados.0.smithi110.stdout:3572: finishing copy_from racing read to smithi11075562-50 2024-09-14T03:49:51.920 INFO:tasks.rados.rados.0.smithi110.stdout:3572: finishing copy_from to smithi11075562-50 2024-09-14T03:49:51.920 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 2483 (ObjNum 1094 snap 345 seq_num 1094) dirty exists 2024-09-14T03:49:51.920 INFO:tasks.rados.rados.0.smithi110.stdout:3570: finishing copy_from racing read to smithi11075562-13 2024-09-14T03:49:51.920 INFO:tasks.rados.rados.0.smithi110.stdout:3570: finishing copy_from to smithi11075562-13 2024-09-14T03:49:51.920 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 2031 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:49:51.920 INFO:tasks.rados.rados.0.smithi110.stdout:3573: finishing write tid 1 to smithi11075562-4 2024-09-14T03:49:51.922 INFO:tasks.rados.rados.0.smithi110.stdout:3573: finishing write tid 2 to smithi11075562-4 2024-09-14T03:49:51.922 INFO:tasks.rados.rados.0.smithi110.stdout:3573: finishing write tid 3 to smithi11075562-4 2024-09-14T03:49:51.923 INFO:tasks.rados.rados.0.smithi110.stdout:3573: finishing write tid 4 to smithi11075562-4 2024-09-14T03:49:51.924 INFO:tasks.rados.rados.0.smithi110.stdout:3573: finishing write tid 5 to smithi11075562-4 2024-09-14T03:49:51.924 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2161 (ObjNum 1107 snap 351 seq_num 1107) dirty exists 2024-09-14T03:49:51.924 INFO:tasks.rados.rados.0.smithi110.stdout:3573: left oid 4 (ObjNum 1107 snap 351 seq_num 1107) 2024-09-14T03:49:52.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:51 smithi110 ceph-mon[88262]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T03:49:52.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:51 smithi110 ceph-mon[86423]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T03:49:52.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:51 smithi195 ceph-mon[82388]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T03:49:52.855 INFO:tasks.rados.rados.0.smithi110.stdout:3570: done (4 left) 2024-09-14T03:49:52.855 INFO:tasks.rados.rados.0.smithi110.stdout:3571: done (3 left) 2024-09-14T03:49:52.855 INFO:tasks.rados.rados.0.smithi110.stdout:3572: done (2 left) 2024-09-14T03:49:52.855 INFO:tasks.rados.rados.0.smithi110.stdout:3573: done (1 left) 2024-09-14T03:49:52.855 INFO:tasks.rados.rados.0.smithi110.stdout:3574: done (0 left) 2024-09-14T03:49:52.856 INFO:tasks.rados.rados.0.smithi110.stdout:3575: read oid 10 snap -1 2024-09-14T03:49:52.856 INFO:tasks.rados.rados.0.smithi110.stdout:3575: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:49:52.856 INFO:tasks.rados.rados.0.smithi110.stdout:3576: snap_create 2024-09-14T03:49:52.883 INFO:tasks.rados.rados.0.smithi110.stdout:3575: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:49:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:52 smithi110 ceph-mon[88262]: pgmap v805: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:49:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:52 smithi110 ceph-mon[88262]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T03:49:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:52 smithi110 ceph-mon[86423]: pgmap v805: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:49:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:52 smithi110 ceph-mon[86423]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T03:49:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:52 smithi195 ceph-mon[82388]: pgmap v805: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:49:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:52 smithi195 ceph-mon[82388]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T03:49:53.816 INFO:tasks.rados.rados.0.smithi110.stdout:3575: done (1 left) 2024-09-14T03:49:53.816 INFO:tasks.rados.rados.0.smithi110.stdout:3576: done (0 left) 2024-09-14T03:49:53.816 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 353 2024-09-14T03:49:53.816 INFO:tasks.rados.rados.0.smithi110.stdout:3577: seq_num 1108 ranges {3817472=565248,4382720=647168,5029888=483328,5513216=73728} 2024-09-14T03:49:53.840 INFO:tasks.rados.rados.0.smithi110.stdout:3577: writing smithi11075562-42 from 3817472 to 4382720 tid 1 2024-09-14T03:49:53.844 INFO:tasks.rados.rados.0.smithi110.stdout:3577: writing smithi11075562-42 from 4382720 to 5029888 tid 2 2024-09-14T03:49:53.847 INFO:tasks.rados.rados.0.smithi110.stdout:3577: writing smithi11075562-42 from 5029888 to 5513216 tid 3 2024-09-14T03:49:53.848 INFO:tasks.rados.rados.0.smithi110.stdout:3577: writing smithi11075562-42 from 5513216 to 5586944 tid 4 2024-09-14T03:49:53.848 INFO:tasks.rados.rados.0.smithi110.stdout:3578: setattr oid 12 current snap is 353 2024-09-14T03:49:53.850 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 353 2024-09-14T03:49:53.850 INFO:tasks.rados.rados.0.smithi110.stdout:3579: seq_num 1109 ranges {0=581632,581632=532480,1114112=647168,1761280=24576} 2024-09-14T03:49:53.853 INFO:tasks.rados.rados.0.smithi110.stdout:3579: writing smithi11075562-25 from 0 to 581632 tid 1 2024-09-14T03:49:53.856 INFO:tasks.rados.rados.0.smithi110.stdout:3579: writing smithi11075562-25 from 581632 to 1114112 tid 2 2024-09-14T03:49:53.862 INFO:tasks.rados.rados.0.smithi110.stdout:3579: writing smithi11075562-25 from 1114112 to 1761280 tid 3 2024-09-14T03:49:53.862 INFO:tasks.rados.rados.0.smithi110.stdout:3579: writing smithi11075562-25 from 1761280 to 1785856 tid 4 2024-09-14T03:49:53.862 INFO:tasks.rados.rados.0.smithi110.stdout:3580: read oid 23 snap -1 2024-09-14T03:49:53.862 INFO:tasks.rados.rados.0.smithi110.stdout:3580: expect (ObjNum 1065 snap 336 seq_num 1065) 2024-09-14T03:49:53.862 INFO:tasks.rados.rados.0.smithi110.stdout:3581: delete oid 17 current snap is 353 2024-09-14T03:49:53.863 INFO:tasks.rados.rados.0.smithi110.stdout:3577: finishing write tid 1 to smithi11075562-42 2024-09-14T03:49:53.863 INFO:tasks.rados.rados.0.smithi110.stdout:3577: finishing write tid 2 to smithi11075562-42 2024-09-14T03:49:53.863 INFO:tasks.rados.rados.0.smithi110.stdout:3577: finishing write tid 3 to smithi11075562-42 2024-09-14T03:49:53.863 INFO:tasks.rados.rados.0.smithi110.stdout:3577: finishing write tid 4 to smithi11075562-42 2024-09-14T03:49:53.863 INFO:tasks.rados.rados.0.smithi110.stdout:3577: finishing write tid 5 to smithi11075562-42 2024-09-14T03:49:53.864 INFO:tasks.rados.rados.0.smithi110.stdout:3577: finishing write tid 6 to smithi11075562-42 2024-09-14T03:49:53.864 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2204 (ObjNum 1108 snap 353 seq_num 1108) dirty exists 2024-09-14T03:49:53.864 INFO:tasks.rados.rados.0.smithi110.stdout:3577: left oid 42 (ObjNum 1108 snap 353 seq_num 1108) 2024-09-14T03:49:53.865 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2488 (ObjNum 1063 snap 336 seq_num 1063) dirty exists 2024-09-14T03:49:53.865 INFO:tasks.rados.rados.0.smithi110.stdout:3579: finishing write tid 1 to smithi11075562-25 2024-09-14T03:49:53.867 INFO:tasks.rados.rados.0.smithi110.stdout:3577: done (4 left) 2024-09-14T03:49:53.867 INFO:tasks.rados.rados.0.smithi110.stdout:3578: done (3 left) 2024-09-14T03:49:53.868 INFO:tasks.rados.rados.0.smithi110.stdout:3581: done (2 left) 2024-09-14T03:49:53.868 INFO:tasks.rados.rados.0.smithi110.stdout:3582: setattr oid 46 current snap is 353 2024-09-14T03:49:53.869 INFO:tasks.rados.rados.0.smithi110.stdout:3583: delete oid 33 current snap is 353 2024-09-14T03:49:53.870 INFO:tasks.rados.rados.0.smithi110.stdout:3579: finishing write tid 2 to smithi11075562-25 2024-09-14T03:49:53.873 INFO:tasks.rados.rados.0.smithi110.stdout:3579: finishing write tid 3 to smithi11075562-25 2024-09-14T03:49:53.873 INFO:tasks.rados.rados.0.smithi110.stdout:3579: finishing write tid 4 to smithi11075562-25 2024-09-14T03:49:53.874 INFO:tasks.rados.rados.0.smithi110.stdout:3579: finishing write tid 5 to smithi11075562-25 2024-09-14T03:49:53.875 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 2145 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-14T03:49:53.877 INFO:tasks.rados.rados.0.smithi110.stdout:3579: finishing write tid 6 to smithi11075562-25 2024-09-14T03:49:53.877 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2493 (ObjNum 1109 snap 353 seq_num 1109) dirty exists 2024-09-14T03:49:53.877 INFO:tasks.rados.rados.0.smithi110.stdout:3579: left oid 25 (ObjNum 1109 snap 353 seq_num 1109) 2024-09-14T03:49:53.877 INFO:tasks.rados.rados.0.smithi110.stdout:3579: done (3 left) 2024-09-14T03:49:53.878 INFO:tasks.rados.rados.0.smithi110.stdout:3582: done (2 left) 2024-09-14T03:49:53.878 INFO:tasks.rados.rados.0.smithi110.stdout:3583: done (1 left) 2024-09-14T03:49:53.878 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 353 2024-09-14T03:49:53.878 INFO:tasks.rados.rados.0.smithi110.stdout:3584: seq_num 1112 ranges {1646592=630784,2277376=638976,2916352=466944,3383296=98304} 2024-09-14T03:49:53.889 INFO:tasks.rados.rados.0.smithi110.stdout:3584: writing smithi11075562-29 from 1646592 to 2277376 tid 1 2024-09-14T03:49:53.893 INFO:tasks.rados.rados.0.smithi110.stdout:3584: writing smithi11075562-29 from 2277376 to 2916352 tid 2 2024-09-14T03:49:53.896 INFO:tasks.rados.rados.0.smithi110.stdout:3584: writing smithi11075562-29 from 2916352 to 3383296 tid 3 2024-09-14T03:49:53.896 INFO:tasks.rados.rados.0.smithi110.stdout:3584: writing smithi11075562-29 from 3383296 to 3481600 tid 4 2024-09-14T03:49:53.897 INFO:tasks.rados.rados.0.smithi110.stdout:3585: snap_remove snap 347 2024-09-14T03:49:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:53 smithi110 ceph-mon[88262]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T03:49:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:53 smithi110 ceph-mon[88262]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T03:49:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:53 smithi110 ceph-mon[86423]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T03:49:54.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:53 smithi110 ceph-mon[86423]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T03:49:54.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:53 smithi195 ceph-mon[82388]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T03:49:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:53 smithi195 ceph-mon[82388]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T03:49:54.867 INFO:tasks.rados.rados.0.smithi110.stdout:3585: done (2 left) 2024-09-14T03:49:54.867 INFO:tasks.rados.rados.0.smithi110.stdout:3586: copy_from oid 36 from oid 28 current snap is 353 2024-09-14T03:49:54.867 INFO:tasks.rados.rados.0.smithi110.stdout:3580: expect (ObjNum 1065 snap 336 seq_num 1065) 2024-09-14T03:49:55.472 INFO:tasks.rados.rados.0.smithi110.stdout:3584: finishing write tid 1 to smithi11075562-29 2024-09-14T03:49:55.472 INFO:tasks.rados.rados.0.smithi110.stdout:3584: finishing write tid 2 to smithi11075562-29 2024-09-14T03:49:55.472 INFO:tasks.rados.rados.0.smithi110.stdout:3584: finishing write tid 3 to smithi11075562-29 2024-09-14T03:49:55.472 INFO:tasks.rados.rados.0.smithi110.stdout:3584: finishing write tid 4 to smithi11075562-29 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:3584: finishing write tid 5 to smithi11075562-29 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:3584: finishing write tid 6 to smithi11075562-29 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1645 (ObjNum 1112 snap 353 seq_num 1112) dirty exists 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:3584: left oid 29 (ObjNum 1112 snap 353 seq_num 1112) 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:3580: done (2 left) 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:3584: done (1 left) 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:3587: rollback oid 20 current snap is 353 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 349 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:3586: finishing copy_from racing read to smithi11075562-36 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:3586: finishing copy_from to smithi11075562-36 2024-09-14T03:49:55.473 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2141 (ObjNum 1077 snap 340 seq_num 1077) dirty exists 2024-09-14T03:49:55.474 INFO:tasks.rados.rados.0.smithi110.stdout:3586: done (1 left) 2024-09-14T03:49:55.474 INFO:tasks.rados.rados.0.smithi110.stdout:3588: snap_create 2024-09-14T03:49:55.474 INFO:tasks.rados.rados.0.smithi110.stdout:3587: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:49:55.474 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 0 (ObjNum 49 snap 0 seq_num 65147344) dirty dne 2024-09-14T03:49:55.867 INFO:tasks.rados.rados.0.smithi110.stdout:3587: done (1 left) 2024-09-14T03:49:55.867 INFO:tasks.rados.rados.0.smithi110.stdout:3588: done (0 left) 2024-09-14T03:49:55.867 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 354 2024-09-14T03:49:55.867 INFO:tasks.rados.rados.0.smithi110.stdout:3589: seq_num 1113 ranges {11116544=647168,11763712=524288,12288000=622592,12910592=57344} 2024-09-14T03:49:55.929 INFO:tasks.rados.rados.0.smithi110.stdout:3589: writing smithi11075562-4 from 11116544 to 11763712 tid 1 2024-09-14T03:49:55.932 INFO:tasks.rados.rados.0.smithi110.stdout:3589: writing smithi11075562-4 from 11763712 to 12288000 tid 2 2024-09-14T03:49:55.936 INFO:tasks.rados.rados.0.smithi110.stdout:3589: writing smithi11075562-4 from 12288000 to 12910592 tid 3 2024-09-14T03:49:55.936 INFO:tasks.rados.rados.0.smithi110.stdout:3589: writing smithi11075562-4 from 12910592 to 12967936 tid 4 2024-09-14T03:49:55.936 INFO:tasks.rados.rados.0.smithi110.stdout:3590: snap_remove snap 351 2024-09-14T03:49:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:55 smithi110 ceph-mon[88262]: pgmap v809: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:49:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:55 smithi110 ceph-mon[88262]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T03:49:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:55 smithi110 ceph-mon[86423]: pgmap v809: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:49:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:55 smithi110 ceph-mon[86423]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T03:49:56.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:55 smithi195 ceph-mon[82388]: pgmap v809: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:49:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:55 smithi195 ceph-mon[82388]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T03:49:56.875 INFO:tasks.rados.rados.0.smithi110.stdout:3590: done (1 left) 2024-09-14T03:49:56.875 INFO:tasks.rados.rados.0.smithi110.stdout:3591: snap_remove snap 353 2024-09-14T03:49:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:56 smithi110 ceph-mon[88262]: pgmap v811: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.4 MiB/s wr, 25 op/s 2024-09-14T03:49:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:56 smithi110 ceph-mon[88262]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T03:49:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:56 smithi110 ceph-mon[86423]: pgmap v811: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.4 MiB/s wr, 25 op/s 2024-09-14T03:49:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:56 smithi110 ceph-mon[86423]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T03:49:57.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:56 smithi195 ceph-mon[82388]: pgmap v811: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.4 MiB/s wr, 25 op/s 2024-09-14T03:49:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:56 smithi195 ceph-mon[82388]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T03:49:57.880 INFO:tasks.rados.rados.0.smithi110.stdout:3589: finishing write tid 1 to smithi11075562-4 2024-09-14T03:49:57.880 INFO:tasks.rados.rados.0.smithi110.stdout:3589: finishing write tid 2 to smithi11075562-4 2024-09-14T03:49:57.880 INFO:tasks.rados.rados.0.smithi110.stdout:3589: finishing write tid 3 to smithi11075562-4 2024-09-14T03:49:57.880 INFO:tasks.rados.rados.0.smithi110.stdout:3589: finishing write tid 4 to smithi11075562-4 2024-09-14T03:49:57.880 INFO:tasks.rados.rados.0.smithi110.stdout:3589: finishing write tid 5 to smithi11075562-4 2024-09-14T03:49:57.881 INFO:tasks.rados.rados.0.smithi110.stdout:3589: finishing write tid 6 to smithi11075562-4 2024-09-14T03:49:57.881 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2166 (ObjNum 1113 snap 354 seq_num 1113) dirty exists 2024-09-14T03:49:57.881 INFO:tasks.rados.rados.0.smithi110.stdout:3589: left oid 4 (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:49:57.881 INFO:tasks.rados.rados.0.smithi110.stdout:3589: done (1 left) 2024-09-14T03:49:57.881 INFO:tasks.rados.rados.0.smithi110.stdout:3591: done (0 left) 2024-09-14T03:49:57.881 INFO:tasks.rados.rados.0.smithi110.stdout:3592: delete oid 49 current snap is 354 2024-09-14T03:49:57.885 INFO:tasks.rados.rados.0.smithi110.stdout:3592: done (0 left) 2024-09-14T03:49:57.885 INFO:tasks.rados.rados.0.smithi110.stdout:3593: snap_remove snap 350 2024-09-14T03:49:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:57 smithi110 ceph-mon[88262]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T03:49:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:57 smithi110 ceph-mon[86423]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T03:49:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:58.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:57 smithi195 ceph-mon[82388]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T03:49:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:49:58.818 INFO:tasks.rados.rados.0.smithi110.stdout:3593: done (0 left) 2024-09-14T03:49:58.818 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 50 current snap is 354 2024-09-14T03:49:58.818 INFO:tasks.rados.rados.0.smithi110.stdout:3594: seq_num 1115 ranges {1531904=679936,2211840=688128,2899968=516096} 2024-09-14T03:49:58.830 INFO:tasks.rados.rados.0.smithi110.stdout:3594: writing smithi11075562-50 from 1531904 to 2211840 tid 1 2024-09-14T03:49:58.834 INFO:tasks.rados.rados.0.smithi110.stdout:3594: writing smithi11075562-50 from 2211840 to 2899968 tid 2 2024-09-14T03:49:58.838 INFO:tasks.rados.rados.0.smithi110.stdout:3594: writing smithi11075562-50 from 2899968 to 3416064 tid 3 2024-09-14T03:49:58.838 INFO:tasks.rados.rados.0.smithi110.stdout:3595: delete oid 42 current snap is 354 2024-09-14T03:49:58.838 INFO:tasks.rados.rados.0.smithi110.stdout:3594: finishing write tid 1 to smithi11075562-50 2024-09-14T03:49:58.841 INFO:tasks.rados.rados.0.smithi110.stdout:3594: finishing write tid 2 to smithi11075562-50 2024-09-14T03:49:58.841 INFO:tasks.rados.rados.0.smithi110.stdout:3594: finishing write tid 3 to smithi11075562-50 2024-09-14T03:49:58.842 INFO:tasks.rados.rados.0.smithi110.stdout:3595: done (1 left) 2024-09-14T03:49:58.842 INFO:tasks.rados.rados.0.smithi110.stdout:3596: snap_create 2024-09-14T03:49:58.842 INFO:tasks.rados.rados.0.smithi110.stdout:3594: finishing write tid 4 to smithi11075562-50 2024-09-14T03:49:58.843 INFO:tasks.rados.rados.0.smithi110.stdout:3594: finishing write tid 5 to smithi11075562-50 2024-09-14T03:49:58.843 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 2487 (ObjNum 1115 snap 354 seq_num 1115) dirty exists 2024-09-14T03:49:58.843 INFO:tasks.rados.rados.0.smithi110.stdout:3594: left oid 50 (ObjNum 1115 snap 354 seq_num 1115) 2024-09-14T03:49:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:58 smithi110 ceph-mon[88262]: pgmap v814: 121 pgs: 121 active+clean; 198 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.1 MiB/s wr, 21 op/s 2024-09-14T03:49:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:58 smithi110 ceph-mon[88262]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T03:49:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:49:58 smithi110 ceph-mon[88262]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T03:49:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:58 smithi110 ceph-mon[86423]: pgmap v814: 121 pgs: 121 active+clean; 198 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.1 MiB/s wr, 21 op/s 2024-09-14T03:49:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:58 smithi110 ceph-mon[86423]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T03:49:59.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:49:58 smithi110 ceph-mon[86423]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T03:49:59.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:58 smithi195 ceph-mon[82388]: pgmap v814: 121 pgs: 121 active+clean; 198 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.1 MiB/s wr, 21 op/s 2024-09-14T03:49:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:58 smithi195 ceph-mon[82388]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T03:49:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:49:58 smithi195 ceph-mon[82388]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T03:49:59.820 INFO:tasks.rados.rados.0.smithi110.stdout:3594: done (1 left) 2024-09-14T03:49:59.820 INFO:tasks.rados.rados.0.smithi110.stdout:3596: done (0 left) 2024-09-14T03:49:59.820 INFO:tasks.rados.rados.0.smithi110.stdout:3597: snap_create 2024-09-14T03:50:00.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:50:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:50:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:00.825 INFO:tasks.rados.rados.0.smithi110.stdout:3597: done (0 left) 2024-09-14T03:50:00.825 INFO:tasks.rados.rados.0.smithi110.stdout:3598: rollback oid 11 current snap is 356 2024-09-14T03:50:00.825 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 354 2024-09-14T03:50:00.825 INFO:tasks.rados.rados.0.smithi110.stdout:3599: rollback oid 32 current snap is 356 2024-09-14T03:50:00.825 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 32 to 354 2024-09-14T03:50:00.825 INFO:tasks.rados.rados.0.smithi110.stdout:3600: delete oid 15 current snap is 356 2024-09-14T03:50:00.831 INFO:tasks.rados.rados.0.smithi110.stdout:3598: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:50:00.832 INFO:tasks.rados.rados.0.smithi110.stdout:3598: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:50:00.832 INFO:tasks.rados.rados.0.smithi110.stdout:3598: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:50:00.832 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 2012 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:50:00.834 INFO:tasks.rados.rados.0.smithi110.stdout:3599: finishing rollback tid 0 to smithi11075562-32 2024-09-14T03:50:00.834 INFO:tasks.rados.rados.0.smithi110.stdout:3599: finishing rollback tid 1 to smithi11075562-32 2024-09-14T03:50:00.834 INFO:tasks.rados.rados.0.smithi110.stdout:3599: finishing rollback tid 2 to smithi11075562-32 2024-09-14T03:50:00.834 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2490 (ObjNum 1094 snap 345 seq_num 1094) dirty exists 2024-09-14T03:50:00.835 INFO:tasks.rados.rados.0.smithi110.stdout:3598: done (2 left) 2024-09-14T03:50:00.835 INFO:tasks.rados.rados.0.smithi110.stdout:3599: done (1 left) 2024-09-14T03:50:00.835 INFO:tasks.rados.rados.0.smithi110.stdout:3600: done (0 left) 2024-09-14T03:50:00.835 INFO:tasks.rados.rados.0.smithi110.stdout:3601: rollback oid 45 current snap is 356 2024-09-14T03:50:00.835 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 345 2024-09-14T03:50:00.835 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 40 current snap is 356 2024-09-14T03:50:00.835 INFO:tasks.rados.rados.0.smithi110.stdout:3602: seq_num 1118 ranges {0=729088,729088=794624,1523712=409600} 2024-09-14T03:50:00.839 INFO:tasks.rados.rados.0.smithi110.stdout:3602: writing smithi11075562-40 from 0 to 729088 tid 1 2024-09-14T03:50:00.844 INFO:tasks.rados.rados.0.smithi110.stdout:3602: writing smithi11075562-40 from 729088 to 1523712 tid 2 2024-09-14T03:50:00.847 INFO:tasks.rados.rados.0.smithi110.stdout:3602: writing smithi11075562-40 from 1523712 to 1933312 tid 3 2024-09-14T03:50:00.847 INFO:tasks.rados.rados.0.smithi110.stdout:3603: snap_create 2024-09-14T03:50:00.847 INFO:tasks.rados.rados.0.smithi110.stdout:3601: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:50:00.847 INFO:tasks.rados.rados.0.smithi110.stdout:3601: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:50:00.847 INFO:tasks.rados.rados.0.smithi110.stdout:3601: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:50:00.847 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2509 (ObjNum 1084 snap 343 seq_num 1084) dirty exists 2024-09-14T03:50:00.847 INFO:tasks.rados.rados.0.smithi110.stdout:3602: finishing write tid 1 to smithi11075562-40 2024-09-14T03:50:00.851 INFO:tasks.rados.rados.0.smithi110.stdout:3602: finishing write tid 2 to smithi11075562-40 2024-09-14T03:50:00.851 INFO:tasks.rados.rados.0.smithi110.stdout:3602: finishing write tid 3 to smithi11075562-40 2024-09-14T03:50:00.851 INFO:tasks.rados.rados.0.smithi110.stdout:3602: finishing write tid 4 to smithi11075562-40 2024-09-14T03:50:00.852 INFO:tasks.rados.rados.0.smithi110.stdout:3602: finishing write tid 5 to smithi11075562-40 2024-09-14T03:50:00.852 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 40 v 2163 (ObjNum 1118 snap 356 seq_num 1118) dirty exists 2024-09-14T03:50:00.852 INFO:tasks.rados.rados.0.smithi110.stdout:3602: left oid 40 (ObjNum 1118 snap 356 seq_num 1118) 2024-09-14T03:50:01.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:00 smithi110 ceph-mon[86423]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T03:50:01.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:00 smithi110 ceph-mon[86423]: overall HEALTH_OK 2024-09-14T03:50:01.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:00 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:50:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:00 smithi110 ceph-mon[88262]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T03:50:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:00 smithi110 ceph-mon[88262]: overall HEALTH_OK 2024-09-14T03:50:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:00 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:50:01.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:00 smithi195 ceph-mon[82388]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T03:50:01.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:00 smithi195 ceph-mon[82388]: overall HEALTH_OK 2024-09-14T03:50:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:00 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:50:01.821 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:50:01] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-14T03:50:01.829 INFO:tasks.rados.rados.0.smithi110.stdout:3601: done (2 left) 2024-09-14T03:50:01.829 INFO:tasks.rados.rados.0.smithi110.stdout:3602: done (1 left) 2024-09-14T03:50:01.829 INFO:tasks.rados.rados.0.smithi110.stdout:3603: done (0 left) 2024-09-14T03:50:01.829 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 357 2024-09-14T03:50:01.829 INFO:tasks.rados.rados.0.smithi110.stdout:3604: seq_num 1119 ranges {6643712=745472,7389184=679936,8069120=524288} 2024-09-14T03:50:01.868 INFO:tasks.rados.rados.0.smithi110.stdout:3604: writing smithi11075562-13 from 6643712 to 7389184 tid 1 2024-09-14T03:50:01.873 INFO:tasks.rados.rados.0.smithi110.stdout:3604: writing smithi11075562-13 from 7389184 to 8069120 tid 2 2024-09-14T03:50:01.876 INFO:tasks.rados.rados.0.smithi110.stdout:3604: writing smithi11075562-13 from 8069120 to 8593408 tid 3 2024-09-14T03:50:01.876 INFO:tasks.rados.rados.0.smithi110.stdout:3605: rollback oid 30 current snap is 357 2024-09-14T03:50:01.876 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 356 2024-09-14T03:50:01.877 INFO:tasks.rados.rados.0.smithi110.stdout:3604: finishing write tid 1 to smithi11075562-13 2024-09-14T03:50:01.877 INFO:tasks.rados.rados.0.smithi110.stdout:3606: snap_create 2024-09-14T03:50:01.879 INFO:tasks.rados.rados.0.smithi110.stdout:3604: finishing write tid 2 to smithi11075562-13 2024-09-14T03:50:01.881 INFO:tasks.rados.rados.0.smithi110.stdout:3604: finishing write tid 3 to smithi11075562-13 2024-09-14T03:50:01.881 INFO:tasks.rados.rados.0.smithi110.stdout:3604: finishing write tid 4 to smithi11075562-13 2024-09-14T03:50:01.881 INFO:tasks.rados.rados.0.smithi110.stdout:3605: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:50:01.881 INFO:tasks.rados.rados.0.smithi110.stdout:3605: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:50:01.881 INFO:tasks.rados.rados.0.smithi110.stdout:3605: finishing rollback tid 2 to smithi11075562-30 2024-09-14T03:50:01.882 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 2494 (ObjNum 1092 snap 343 seq_num 1092) dirty exists 2024-09-14T03:50:01.882 INFO:tasks.rados.rados.0.smithi110.stdout:3604: finishing write tid 5 to smithi11075562-13 2024-09-14T03:50:01.882 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 2220 (ObjNum 1119 snap 357 seq_num 1119) dirty exists 2024-09-14T03:50:01.882 INFO:tasks.rados.rados.0.smithi110.stdout:3604: left oid 13 (ObjNum 1119 snap 357 seq_num 1119) 2024-09-14T03:50:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: pgmap v818: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 3.4 MiB/s wr, 12 op/s 2024-09-14T03:50:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T03:50:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:50:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: pgmap v818: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 3.4 MiB/s wr, 12 op/s 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:50:02.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: pgmap v818: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 3.4 MiB/s wr, 12 op/s 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:50:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:50:02.835 INFO:tasks.rados.rados.0.smithi110.stdout:3604: done (2 left) 2024-09-14T03:50:02.835 INFO:tasks.rados.rados.0.smithi110.stdout:3605: done (1 left) 2024-09-14T03:50:02.835 INFO:tasks.rados.rados.0.smithi110.stdout:3606: done (0 left) 2024-09-14T03:50:02.835 INFO:tasks.rados.rados.0.smithi110.stdout:3607: read oid 41 snap -1 2024-09-14T03:50:02.835 INFO:tasks.rados.rados.0.smithi110.stdout:3607: expect (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:50:02.836 INFO:tasks.rados.rados.0.smithi110.stdout:3608: delete oid 28 current snap is 358 2024-09-14T03:50:02.840 INFO:tasks.rados.rados.0.smithi110.stdout:3608: done (1 left) 2024-09-14T03:50:02.840 INFO:tasks.rados.rados.0.smithi110.stdout:3609: snap_create 2024-09-14T03:50:02.866 INFO:tasks.rados.rados.0.smithi110.stdout:3607: expect (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:50:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:02 smithi110 ceph-mon[86423]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T03:50:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:02 smithi110 ceph-mon[88262]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T03:50:03.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:02 smithi195 ceph-mon[82388]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T03:50:03.818 INFO:tasks.rados.rados.0.smithi110.stdout:3607: done (1 left) 2024-09-14T03:50:03.818 INFO:tasks.rados.rados.0.smithi110.stdout:3609: done (0 left) 2024-09-14T03:50:03.818 INFO:tasks.rados.rados.0.smithi110.stdout:3610: snap_remove snap 357 2024-09-14T03:50:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:03 smithi110 ceph-mon[88262]: pgmap v821: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 774 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-14T03:50:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:03 smithi110 ceph-mon[88262]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T03:50:04.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:03 smithi110 ceph-mon[88262]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T03:50:04.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:03 smithi110 ceph-mon[86423]: pgmap v821: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 774 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-14T03:50:04.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:03 smithi110 ceph-mon[86423]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T03:50:04.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:03 smithi110 ceph-mon[86423]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T03:50:04.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:03 smithi195 ceph-mon[82388]: pgmap v821: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 774 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-14T03:50:04.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:03 smithi195 ceph-mon[82388]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T03:50:04.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:03 smithi195 ceph-mon[82388]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T03:50:04.821 INFO:tasks.rados.rados.0.smithi110.stdout:3610: done (0 left) 2024-09-14T03:50:04.821 INFO:tasks.rados.rados.0.smithi110.stdout:3611: snap_create 2024-09-14T03:50:05.828 INFO:tasks.rados.rados.0.smithi110.stdout:3611: done (0 left) 2024-09-14T03:50:05.828 INFO:tasks.rados.rados.0.smithi110.stdout:3612: copy_from oid 13 from oid 23 current snap is 360 2024-09-14T03:50:05.828 INFO:tasks.rados.rados.0.smithi110.stdout:3613: delete oid 7 current snap is 360 2024-09-14T03:50:05.831 INFO:tasks.rados.rados.0.smithi110.stdout:3613: done (1 left) 2024-09-14T03:50:05.831 INFO:tasks.rados.rados.0.smithi110.stdout:3614: snap_create 2024-09-14T03:50:05.877 INFO:tasks.rados.rados.0.smithi110.stdout:3612: finishing copy_from racing read to smithi11075562-13 2024-09-14T03:50:05.885 INFO:tasks.rados.rados.0.smithi110.stdout:3612: finishing copy_from to smithi11075562-13 2024-09-14T03:50:05.885 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 2021 (ObjNum 1065 snap 336 seq_num 1065) dirty exists 2024-09-14T03:50:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:05 smithi110 ceph-mon[88262]: pgmap v824: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 472 KiB/s wr, 2 op/s 2024-09-14T03:50:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:05 smithi110 ceph-mon[88262]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T03:50:06.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:05 smithi110 ceph-mon[86423]: pgmap v824: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 472 KiB/s wr, 2 op/s 2024-09-14T03:50:06.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:05 smithi110 ceph-mon[86423]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T03:50:06.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:05 smithi195 ceph-mon[82388]: pgmap v824: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 472 KiB/s wr, 2 op/s 2024-09-14T03:50:06.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:05 smithi195 ceph-mon[82388]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T03:50:06.832 INFO:tasks.rados.rados.0.smithi110.stdout:3612: done (1 left) 2024-09-14T03:50:06.832 INFO:tasks.rados.rados.0.smithi110.stdout:3614: done (0 left) 2024-09-14T03:50:06.832 INFO:tasks.rados.rados.0.smithi110.stdout:3615: snap_create 2024-09-14T03:50:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:06 smithi110 ceph-mon[88262]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T03:50:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:06 smithi110 ceph-mon[86423]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T03:50:07.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:06 smithi195 ceph-mon[82388]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T03:50:07.833 INFO:tasks.rados.rados.0.smithi110.stdout:3615: done (0 left) 2024-09-14T03:50:07.834 INFO:tasks.rados.rados.0.smithi110.stdout:3616: snap_create 2024-09-14T03:50:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:07 smithi110 ceph-mon[88262]: pgmap v827: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 2 op/s 2024-09-14T03:50:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:07 smithi110 ceph-mon[88262]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T03:50:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:07 smithi110 ceph-mon[86423]: pgmap v827: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 2 op/s 2024-09-14T03:50:08.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:07 smithi110 ceph-mon[86423]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T03:50:08.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:07 smithi195 ceph-mon[82388]: pgmap v827: 121 pgs: 121 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 2 op/s 2024-09-14T03:50:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:07 smithi195 ceph-mon[82388]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T03:50:08.821 INFO:tasks.rados.rados.0.smithi110.stdout:3616: done (0 left) 2024-09-14T03:50:08.822 INFO:tasks.rados.rados.0.smithi110.stdout:3617: snap_create 2024-09-14T03:50:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:08 smithi110 ceph-mon[88262]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T03:50:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:08 smithi110 ceph-mon[88262]: pgmap v830: 121 pgs: 121 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-14T03:50:09.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:08 smithi110 ceph-mon[88262]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T03:50:09.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:08 smithi110 ceph-mon[86423]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T03:50:09.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:08 smithi110 ceph-mon[86423]: pgmap v830: 121 pgs: 121 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-14T03:50:09.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:08 smithi110 ceph-mon[86423]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T03:50:09.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:08 smithi195 ceph-mon[82388]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T03:50:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:08 smithi195 ceph-mon[82388]: pgmap v830: 121 pgs: 121 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-14T03:50:09.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:08 smithi195 ceph-mon[82388]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T03:50:09.823 INFO:tasks.rados.rados.0.smithi110.stdout:3617: done (0 left) 2024-09-14T03:50:09.823 INFO:tasks.rados.rados.0.smithi110.stdout:3618: read oid 3 snap 362 2024-09-14T03:50:09.823 INFO:tasks.rados.rados.0.smithi110.stdout:3618: expect (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:50:09.823 INFO:tasks.rados.rados.0.smithi110.stdout:3619: snap_create 2024-09-14T03:50:09.833 INFO:tasks.rados.rados.0.smithi110.stdout:3618: expect (ObjNum 1063 snap 336 seq_num 1063) 2024-09-14T03:50:10.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:50:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:50:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:10.828 INFO:tasks.rados.rados.0.smithi110.stdout:3618: done (1 left) 2024-09-14T03:50:10.828 INFO:tasks.rados.rados.0.smithi110.stdout:3619: done (0 left) 2024-09-14T03:50:10.828 INFO:tasks.rados.rados.0.smithi110.stdout:3620: read oid 48 snap 358 2024-09-14T03:50:10.828 INFO:tasks.rados.rados.0.smithi110.stdout:3620: expect deleted 2024-09-14T03:50:10.828 INFO:tasks.rados.rados.0.smithi110.stdout:3621: read oid 39 snap 345 2024-09-14T03:50:10.828 INFO:tasks.rados.rados.0.smithi110.stdout:3621: expect (ObjNum 1090 snap 343 seq_num 1090) 2024-09-14T03:50:10.828 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 13 current snap is 365 2024-09-14T03:50:10.828 INFO:tasks.rados.rados.0.smithi110.stdout:3622: seq_num 1122 ranges {8077312=794624,8871936=729088,9601024=475136} 2024-09-14T03:50:10.885 INFO:tasks.rados.rados.0.smithi110.stdout:3622: writing smithi11075562-13 from 8077312 to 8871936 tid 1 2024-09-14T03:50:10.890 INFO:tasks.rados.rados.0.smithi110.stdout:3622: writing smithi11075562-13 from 8871936 to 9601024 tid 2 2024-09-14T03:50:10.894 INFO:tasks.rados.rados.0.smithi110.stdout:3622: writing smithi11075562-13 from 9601024 to 10076160 tid 3 2024-09-14T03:50:10.894 INFO:tasks.rados.rados.0.smithi110.stdout:3623: snap_create 2024-09-14T03:50:10.894 INFO:tasks.rados.rados.0.smithi110.stdout:3622: finishing write tid 1 to smithi11075562-13 2024-09-14T03:50:10.894 INFO:tasks.rados.rados.0.smithi110.stdout:3622: finishing write tid 2 to smithi11075562-13 2024-09-14T03:50:10.898 INFO:tasks.rados.rados.0.smithi110.stdout:3622: finishing write tid 3 to smithi11075562-13 2024-09-14T03:50:10.898 INFO:tasks.rados.rados.0.smithi110.stdout:3622: finishing write tid 4 to smithi11075562-13 2024-09-14T03:50:10.899 INFO:tasks.rados.rados.0.smithi110.stdout:3622: finishing write tid 5 to smithi11075562-13 2024-09-14T03:50:10.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 13 v 2227 (ObjNum 1122 snap 365 seq_num 1122) dirty exists 2024-09-14T03:50:10.900 INFO:tasks.rados.rados.0.smithi110.stdout:3622: left oid 13 (ObjNum 1122 snap 365 seq_num 1122) 2024-09-14T03:50:10.906 INFO:tasks.rados.rados.0.smithi110.stdout:3621: expect (ObjNum 1090 snap 343 seq_num 1090) 2024-09-14T03:50:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:10 smithi110 ceph-mon[88262]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T03:50:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:10 smithi110 ceph-mon[86423]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T03:50:11.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:10 smithi195 ceph-mon[82388]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T03:50:11.824 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:50:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:50:11] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-14T03:50:12.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:11 smithi110 ceph-mon[88262]: pgmap v833: 121 pgs: 121 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.4 MiB/s wr, 2 op/s 2024-09-14T03:50:12.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:11 smithi110 ceph-mon[88262]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T03:50:12.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:11 smithi110 ceph-mon[86423]: pgmap v833: 121 pgs: 121 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.4 MiB/s wr, 2 op/s 2024-09-14T03:50:12.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:11 smithi110 ceph-mon[86423]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T03:50:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:11 smithi195 ceph-mon[82388]: pgmap v833: 121 pgs: 121 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.4 MiB/s wr, 2 op/s 2024-09-14T03:50:12.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:11 smithi195 ceph-mon[82388]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T03:50:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:12 smithi195 ceph-mon[82388]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T03:50:13.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:12 smithi110 ceph-mon[88262]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T03:50:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:12 smithi110 ceph-mon[86423]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T03:50:13.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:13.488 INFO:tasks.rados.rados.0.smithi110.stdout:3620: done (3 left) 2024-09-14T03:50:13.488 INFO:tasks.rados.rados.0.smithi110.stdout:3621: done (2 left) 2024-09-14T03:50:13.488 INFO:tasks.rados.rados.0.smithi110.stdout:3622: done (1 left) 2024-09-14T03:50:13.488 INFO:tasks.rados.rados.0.smithi110.stdout:3623: done (0 left) 2024-09-14T03:50:13.488 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 23 current snap is 366 2024-09-14T03:50:13.488 INFO:tasks.rados.rados.0.smithi110.stdout:3624: seq_num 1123 ranges {8077312=409600,8486912=614400,9101312=598016,9699328=393216} 2024-09-14T03:50:13.533 INFO:tasks.rados.rados.0.smithi110.stdout:3624: writing smithi11075562-23 from 8077312 to 8486912 tid 1 2024-09-14T03:50:13.537 INFO:tasks.rados.rados.0.smithi110.stdout:3624: writing smithi11075562-23 from 8486912 to 9101312 tid 2 2024-09-14T03:50:13.540 INFO:tasks.rados.rados.0.smithi110.stdout:3624: writing smithi11075562-23 from 9101312 to 9699328 tid 3 2024-09-14T03:50:13.543 INFO:tasks.rados.rados.0.smithi110.stdout:3624: writing smithi11075562-23 from 9699328 to 10092544 tid 4 2024-09-14T03:50:13.543 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 366 2024-09-14T03:50:13.543 INFO:tasks.rados.rados.0.smithi110.stdout:3624: finishing write tid 1 to smithi11075562-23 2024-09-14T03:50:13.543 INFO:tasks.rados.rados.0.smithi110.stdout:3624: finishing write tid 2 to smithi11075562-23 2024-09-14T03:50:13.543 INFO:tasks.rados.rados.0.smithi110.stdout:3625: seq_num 1124 ranges {3268608=434176,3702784=499712,4202496=409600,4612096=663552,5275648=32768} 2024-09-14T03:50:13.562 INFO:tasks.rados.rados.0.smithi110.stdout:3625: writing smithi11075562-3 from 3268608 to 3702784 tid 1 2024-09-14T03:50:13.565 INFO:tasks.rados.rados.0.smithi110.stdout:3625: writing smithi11075562-3 from 3702784 to 4202496 tid 2 2024-09-14T03:50:13.568 INFO:tasks.rados.rados.0.smithi110.stdout:3625: writing smithi11075562-3 from 4202496 to 4612096 tid 3 2024-09-14T03:50:13.573 INFO:tasks.rados.rados.0.smithi110.stdout:3625: writing smithi11075562-3 from 4612096 to 5275648 tid 4 2024-09-14T03:50:13.573 INFO:tasks.rados.rados.0.smithi110.stdout:3625: writing smithi11075562-3 from 5275648 to 5308416 tid 5 2024-09-14T03:50:13.573 INFO:tasks.rados.rados.0.smithi110.stdout:3626: read oid 34 snap -1 2024-09-14T03:50:13.573 INFO:tasks.rados.rados.0.smithi110.stdout:3626: expect deleted 2024-09-14T03:50:13.573 INFO:tasks.rados.rados.0.smithi110.stdout:3627: rollback oid 19 current snap is 366 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 360 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3624: finishing write tid 3 to smithi11075562-23 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3624: finishing write tid 4 to smithi11075562-23 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3624: finishing write tid 5 to smithi11075562-23 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3624: finishing write tid 6 to smithi11075562-23 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 2036 (ObjNum 1123 snap 366 seq_num 1123) dirty exists 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3624: left oid 23 (ObjNum 1123 snap 366 seq_num 1123) 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3625: finishing write tid 1 to smithi11075562-3 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3625: finishing write tid 2 to smithi11075562-3 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3624: done (3 left) 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3628: delete oid 5 current snap is 366 2024-09-14T03:50:13.574 INFO:tasks.rados.rados.0.smithi110.stdout:3625: finishing write tid 3 to smithi11075562-3 2024-09-14T03:50:13.578 INFO:tasks.rados.rados.0.smithi110.stdout:3626: done (3 left) 2024-09-14T03:50:13.578 INFO:tasks.rados.rados.0.smithi110.stdout:3628: done (2 left) 2024-09-14T03:50:13.578 INFO:tasks.rados.rados.0.smithi110.stdout:3629: rollback oid 10 current snap is 366 2024-09-14T03:50:13.578 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 361 2024-09-14T03:50:13.578 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 366 2024-09-14T03:50:13.578 INFO:tasks.rados.rados.0.smithi110.stdout:3630: seq_num 1126 ranges {1662976=466944,2129920=720896,2850816=778240,3629056=106496} 2024-09-14T03:50:13.589 INFO:tasks.rados.rados.0.smithi110.stdout:3630: writing smithi11075562-22 from 1662976 to 2129920 tid 1 2024-09-14T03:50:13.594 INFO:tasks.rados.rados.0.smithi110.stdout:3630: writing smithi11075562-22 from 2129920 to 2850816 tid 2 2024-09-14T03:50:13.601 INFO:tasks.rados.rados.0.smithi110.stdout:3630: writing smithi11075562-22 from 2850816 to 3629056 tid 3 2024-09-14T03:50:13.601 INFO:tasks.rados.rados.0.smithi110.stdout:3630: writing smithi11075562-22 from 3629056 to 3735552 tid 4 2024-09-14T03:50:13.601 INFO:tasks.rados.rados.0.smithi110.stdout:3631: rollback oid 41 current snap is 366 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 41 to 360 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3625: finishing write tid 4 to smithi11075562-3 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3625: finishing write tid 5 to smithi11075562-3 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3625: finishing write tid 6 to smithi11075562-3 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3627: finishing rollback tid 0 to smithi11075562-19 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3627: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3625: finishing write tid 7 to smithi11075562-3 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 2148 (ObjNum 1124 snap 366 seq_num 1124) dirty exists 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3625: left oid 3 (ObjNum 1124 snap 366 seq_num 1124) 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3627: finishing rollback tid 2 to smithi11075562-19 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2166 (ObjNum 1105 snap 350 seq_num 1105) dirty exists 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3629: finishing rollback tid 0 to smithi11075562-10 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3629: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:50:13.602 INFO:tasks.rados.rados.0.smithi110.stdout:3629: finishing rollback tid 2 to smithi11075562-10 2024-09-14T03:50:13.603 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2039 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:50:13.603 INFO:tasks.rados.rados.0.smithi110.stdout:3630: finishing write tid 1 to smithi11075562-22 2024-09-14T03:50:13.603 INFO:tasks.rados.rados.0.smithi110.stdout:3630: finishing write tid 2 to smithi11075562-22 2024-09-14T03:50:13.603 INFO:tasks.rados.rados.0.smithi110.stdout:3625: done (4 left) 2024-09-14T03:50:13.603 INFO:tasks.rados.rados.0.smithi110.stdout:3627: done (3 left) 2024-09-14T03:50:13.603 INFO:tasks.rados.rados.0.smithi110.stdout:3629: done (2 left) 2024-09-14T03:50:13.603 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 38 current snap is 366 2024-09-14T03:50:13.603 INFO:tasks.rados.rados.0.smithi110.stdout:3632: seq_num 1127 ranges {2441216=483328,2924544=598016,3522560=581632,4104192=425984} 2024-09-14T03:50:13.615 INFO:tasks.rados.rados.0.smithi110.stdout:3632: writing smithi11075562-38 from 2441216 to 2924544 tid 1 2024-09-14T03:50:13.619 INFO:tasks.rados.rados.0.smithi110.stdout:3632: writing smithi11075562-38 from 2924544 to 3522560 tid 2 2024-09-14T03:50:13.623 INFO:tasks.rados.rados.0.smithi110.stdout:3632: writing smithi11075562-38 from 3522560 to 4104192 tid 3 2024-09-14T03:50:13.626 INFO:tasks.rados.rados.0.smithi110.stdout:3632: writing smithi11075562-38 from 4104192 to 4530176 tid 4 2024-09-14T03:50:13.626 INFO:tasks.rados.rados.0.smithi110.stdout:3633: snap_create 2024-09-14T03:50:13.626 INFO:tasks.rados.rados.0.smithi110.stdout:3630: finishing write tid 3 to smithi11075562-22 2024-09-14T03:50:13.626 INFO:tasks.rados.rados.0.smithi110.stdout:3630: finishing write tid 4 to smithi11075562-22 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3630: finishing write tid 5 to smithi11075562-22 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3631: finishing rollback tid 0 to smithi11075562-41 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3631: finishing rollback tid 1 to smithi11075562-41 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3631: finishing rollback tid 2 to smithi11075562-41 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 2161 (ObjNum 1008 snap 323 seq_num 1008) dirty exists 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3630: finishing write tid 6 to smithi11075562-22 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2499 (ObjNum 1126 snap 366 seq_num 1126) dirty exists 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3630: left oid 22 (ObjNum 1126 snap 366 seq_num 1126) 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3632: finishing write tid 1 to smithi11075562-38 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3632: finishing write tid 2 to smithi11075562-38 2024-09-14T03:50:13.627 INFO:tasks.rados.rados.0.smithi110.stdout:3632: finishing write tid 3 to smithi11075562-38 2024-09-14T03:50:13.628 INFO:tasks.rados.rados.0.smithi110.stdout:3632: finishing write tid 4 to smithi11075562-38 2024-09-14T03:50:13.628 INFO:tasks.rados.rados.0.smithi110.stdout:3632: finishing write tid 5 to smithi11075562-38 2024-09-14T03:50:13.629 INFO:tasks.rados.rados.0.smithi110.stdout:3632: finishing write tid 6 to smithi11075562-38 2024-09-14T03:50:13.630 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1651 (ObjNum 1127 snap 366 seq_num 1127) dirty exists 2024-09-14T03:50:13.630 INFO:tasks.rados.rados.0.smithi110.stdout:3632: left oid 38 (ObjNum 1127 snap 366 seq_num 1127) 2024-09-14T03:50:13.821 INFO:tasks.rados.rados.0.smithi110.stdout:3630: done (3 left) 2024-09-14T03:50:13.821 INFO:tasks.rados.rados.0.smithi110.stdout:3631: done (2 left) 2024-09-14T03:50:13.822 INFO:tasks.rados.rados.0.smithi110.stdout:3632: done (1 left) 2024-09-14T03:50:13.822 INFO:tasks.rados.rados.0.smithi110.stdout:3633: done (0 left) 2024-09-14T03:50:13.822 INFO:tasks.rados.rados.0.smithi110.stdout:3634: delete oid 49 current snap is 367 2024-09-14T03:50:13.825 INFO:tasks.rados.rados.0.smithi110.stdout:3634: done (0 left) 2024-09-14T03:50:13.825 INFO:tasks.rados.rados.0.smithi110.stdout:3635: snap_remove snap 360 2024-09-14T03:50:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:13 smithi110 ceph-mon[88262]: pgmap v836: 121 pgs: 121 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 488 KiB/s wr, 1 op/s 2024-09-14T03:50:14.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:13 smithi110 ceph-mon[88262]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T03:50:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:13 smithi110 ceph-mon[86423]: pgmap v836: 121 pgs: 121 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 488 KiB/s wr, 1 op/s 2024-09-14T03:50:14.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:13 smithi110 ceph-mon[86423]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T03:50:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:13 smithi195 ceph-mon[82388]: pgmap v836: 121 pgs: 121 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 488 KiB/s wr, 1 op/s 2024-09-14T03:50:14.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:13 smithi195 ceph-mon[82388]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T03:50:14.825 INFO:tasks.rados.rados.0.smithi110.stdout:3635: done (0 left) 2024-09-14T03:50:14.825 INFO:tasks.rados.rados.0.smithi110.stdout:3636: rollback oid 20 current snap is 367 2024-09-14T03:50:14.825 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 20 to 361 2024-09-14T03:50:14.825 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 367 2024-09-14T03:50:14.825 INFO:tasks.rados.rados.0.smithi110.stdout:3637: seq_num 1129 ranges {6643712=516096,7159808=770048,7929856=598016,8527872=237568} 2024-09-14T03:50:14.864 INFO:tasks.rados.rados.0.smithi110.stdout:3637: writing smithi11075562-10 from 6643712 to 7159808 tid 1 2024-09-14T03:50:14.868 INFO:tasks.rados.rados.0.smithi110.stdout:3637: writing smithi11075562-10 from 7159808 to 7929856 tid 2 2024-09-14T03:50:14.872 INFO:tasks.rados.rados.0.smithi110.stdout:3637: writing smithi11075562-10 from 7929856 to 8527872 tid 3 2024-09-14T03:50:14.874 INFO:tasks.rados.rados.0.smithi110.stdout:3637: writing smithi11075562-10 from 8527872 to 8765440 tid 4 2024-09-14T03:50:14.874 INFO:tasks.rados.rados.0.smithi110.stdout:3638: copy_from oid 24 from oid 49 current snap is 367 2024-09-14T03:50:14.874 INFO:tasks.rados.rados.0.smithi110.stdout:3636: finishing rollback tid 1 to smithi11075562-20 2024-09-14T03:50:14.874 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 20 v 0 (ObjNum 65 snap 0 seq_num 1751016300) dirty dne 2024-09-14T03:50:14.874 INFO:tasks.rados.rados.0.smithi110.stdout:3637: finishing write tid 1 to smithi11075562-10 2024-09-14T03:50:14.874 INFO:tasks.rados.rados.0.smithi110.stdout:3637: finishing write tid 2 to smithi11075562-10 2024-09-14T03:50:14.874 INFO:tasks.rados.rados.0.smithi110.stdout:3636: done (2 left) 2024-09-14T03:50:14.875 INFO:tasks.rados.rados.0.smithi110.stdout:3639: rollback oid 22 current snap is 367 2024-09-14T03:50:14.875 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 22 to 364 2024-09-14T03:50:14.875 INFO:tasks.rados.rados.0.smithi110.stdout:3640: snap_remove snap 358 2024-09-14T03:50:15.840 INFO:tasks.rados.rados.0.smithi110.stdout:3640: done (3 left) 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3641: delete oid 30 current snap is 367 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3637: finishing write tid 3 to smithi11075562-10 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3637: finishing write tid 4 to smithi11075562-10 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3637: finishing write tid 5 to smithi11075562-10 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3637: finishing write tid 6 to smithi11075562-10 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2044 (ObjNum 1129 snap 367 seq_num 1129) dirty exists 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3637: left oid 10 (ObjNum 1129 snap 367 seq_num 1129) 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3638: finishing copy_from to smithi11075562-24 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3638: got expected ENOENT (src dne) 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3638: finishing copy_from racing read to smithi11075562-24 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3639: finishing rollback tid 0 to smithi11075562-22 2024-09-14T03:50:15.841 INFO:tasks.rados.rados.0.smithi110.stdout:3639: finishing rollback tid 1 to smithi11075562-22 2024-09-14T03:50:15.842 INFO:tasks.rados.rados.0.smithi110.stdout:3639: finishing rollback tid 2 to smithi11075562-22 2024-09-14T03:50:15.842 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2502 (ObjNum 1102 snap 349 seq_num 1102) dirty exists 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:3637: done (3 left) 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:3638: done (2 left) 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:3639: done (1 left) 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:3641: done (0 left) 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:3642: read oid 27 snap -1 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:3642: expect deleted 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:3643: copy_from oid 25 from oid 17 current snap is 367 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 19 current snap is 367 2024-09-14T03:50:15.852 INFO:tasks.rados.rados.0.smithi110.stdout:3644: seq_num 1131 ranges {3899392=548864,4448256=540672,4988928=565248,5554176=425984,5980160=73728} 2024-09-14T03:50:15.875 INFO:tasks.rados.rados.0.smithi110.stdout:3644: writing smithi11075562-19 from 3899392 to 4448256 tid 1 2024-09-14T03:50:15.878 INFO:tasks.rados.rados.0.smithi110.stdout:3644: writing smithi11075562-19 from 4448256 to 4988928 tid 2 2024-09-14T03:50:15.881 INFO:tasks.rados.rados.0.smithi110.stdout:3644: writing smithi11075562-19 from 4988928 to 5554176 tid 3 2024-09-14T03:50:15.884 INFO:tasks.rados.rados.0.smithi110.stdout:3644: writing smithi11075562-19 from 5554176 to 5980160 tid 4 2024-09-14T03:50:15.884 INFO:tasks.rados.rados.0.smithi110.stdout:3644: writing smithi11075562-19 from 5980160 to 6053888 tid 5 2024-09-14T03:50:15.885 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 367 2024-09-14T03:50:15.885 INFO:tasks.rados.rados.0.smithi110.stdout:3645: seq_num 1132 ranges {0=565248,565248=425984,991232=770048,1761280=409600} 2024-09-14T03:50:15.888 INFO:tasks.rados.rados.0.smithi110.stdout:3645: writing smithi11075562-42 from 0 to 565248 tid 1 2024-09-14T03:50:15.890 INFO:tasks.rados.rados.0.smithi110.stdout:3645: writing smithi11075562-42 from 565248 to 991232 tid 2 2024-09-14T03:50:15.894 INFO:tasks.rados.rados.0.smithi110.stdout:3645: writing smithi11075562-42 from 991232 to 1761280 tid 3 2024-09-14T03:50:15.898 INFO:tasks.rados.rados.0.smithi110.stdout:3645: writing smithi11075562-42 from 1761280 to 2170880 tid 4 2024-09-14T03:50:15.898 INFO:tasks.rados.rados.0.smithi110.stdout:3646: rollback oid 10 current snap is 367 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 355 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3643: finishing copy_from to smithi11075562-25 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3643: got expected ENOENT (src dne) 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3643: finishing copy_from racing read to smithi11075562-25 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3644: finishing write tid 1 to smithi11075562-19 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3644: finishing write tid 2 to smithi11075562-19 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3644: finishing write tid 3 to smithi11075562-19 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3644: finishing write tid 4 to smithi11075562-19 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3644: finishing write tid 5 to smithi11075562-19 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3644: finishing write tid 6 to smithi11075562-19 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3642: done (4 left) 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3643: done (3 left) 2024-09-14T03:50:15.899 INFO:tasks.rados.rados.0.smithi110.stdout:3647: read oid 47 snap -1 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3647: expect (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3648: read oid 40 snap -1 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3648: expect (ObjNum 1118 snap 356 seq_num 1118) 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3649: rollback oid 37 current snap is 367 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 37 to 364 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3644: finishing write tid 7 to smithi11075562-19 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2172 (ObjNum 1131 snap 367 seq_num 1131) dirty exists 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3644: left oid 19 (ObjNum 1131 snap 367 seq_num 1131) 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3645: finishing write tid 1 to smithi11075562-42 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3645: finishing write tid 2 to smithi11075562-42 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3644: done (5 left) 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3650: read oid 25 snap -1 2024-09-14T03:50:15.900 INFO:tasks.rados.rados.0.smithi110.stdout:3650: expect (ObjNum 1109 snap 353 seq_num 1109) 2024-09-14T03:50:15.901 INFO:tasks.rados.rados.0.smithi110.stdout:3651: rollback oid 36 current snap is 367 2024-09-14T03:50:15.901 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 36 to 352 2024-09-14T03:50:15.901 INFO:tasks.rados.rados.0.smithi110.stdout:3652: copy_from oid 16 from oid 3 current snap is 367 2024-09-14T03:50:15.901 INFO:tasks.rados.rados.0.smithi110.stdout:3653: copy_from oid 34 from oid 29 current snap is 367 2024-09-14T03:50:15.901 INFO:tasks.rados.rados.0.smithi110.stdout:3654: delete oid 9 current snap is 367 2024-09-14T03:50:15.903 INFO:tasks.rados.rados.0.smithi110.stdout:3646: finishing rollback tid 0 to smithi11075562-10 2024-09-14T03:50:15.903 INFO:tasks.rados.rados.0.smithi110.stdout:3646: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:3645: finishing write tid 3 to smithi11075562-42 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:3646: finishing rollback tid 2 to smithi11075562-10 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2047 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:3646: done (9 left) 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:3654: done (8 left) 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:3655: read oid 28 snap -1 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:3655: expect deleted 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 367 2024-09-14T03:50:15.904 INFO:tasks.rados.rados.0.smithi110.stdout:3656: seq_num 1134 ranges {1449984=598016,2048000=638976,2686976=737280,3424256=229376} 2024-09-14T03:50:15.916 INFO:tasks.rados.rados.0.smithi110.stdout:3656: writing smithi11075562-43 from 1449984 to 2048000 tid 1 2024-09-14T03:50:15.919 INFO:tasks.rados.rados.0.smithi110.stdout:3656: writing smithi11075562-43 from 2048000 to 2686976 tid 2 2024-09-14T03:50:15.924 INFO:tasks.rados.rados.0.smithi110.stdout:3656: writing smithi11075562-43 from 2686976 to 3424256 tid 3 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:3656: writing smithi11075562-43 from 3424256 to 3653632 tid 4 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:3657: rmattr oid 27 current snap is 367 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:3657: done (10 left) 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:3658: delete oid 27 current snap is 367 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:3649: finishing rollback tid 1 to smithi11075562-37 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 0 (ObjNum 81 snap 0 seq_num 0) dirty dne 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:3645: finishing write tid 4 to smithi11075562-42 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:3645: finishing write tid 5 to smithi11075562-42 2024-09-14T03:50:15.926 INFO:tasks.rados.rados.0.smithi110.stdout:3647: expect (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:50:15.968 INFO:tasks.rados.rados.0.smithi110.stdout:3653: finishing copy_from racing read to smithi11075562-34 2024-09-14T03:50:15.968 INFO:tasks.rados.rados.0.smithi110.stdout:3648: expect (ObjNum 1118 snap 356 seq_num 1118) 2024-09-14T03:50:16.017 INFO:tasks.rados.rados.0.smithi110.stdout:3645: finishing write tid 6 to smithi11075562-42 2024-09-14T03:50:16.017 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2237 (ObjNum 1132 snap 367 seq_num 1132) dirty exists 2024-09-14T03:50:16.017 INFO:tasks.rados.rados.0.smithi110.stdout:3645: left oid 42 (ObjNum 1132 snap 367 seq_num 1132) 2024-09-14T03:50:16.017 INFO:tasks.rados.rados.0.smithi110.stdout:3651: finishing rollback tid 0 to smithi11075562-36 2024-09-14T03:50:16.017 INFO:tasks.rados.rados.0.smithi110.stdout:3645: done (10 left) 2024-09-14T03:50:16.017 INFO:tasks.rados.rados.0.smithi110.stdout:3647: done (9 left) 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3648: done (8 left) 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3649: done (7 left) 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3655: done (6 left) 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3658: done (5 left) 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3659: rmattr oid 20 current snap is 367 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3659: done (5 left) 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3660: copy_from oid 2 from oid 49 current snap is 367 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3653: finishing copy_from to smithi11075562-34 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 1645 (ObjNum 1112 snap 353 seq_num 1112) dirty exists 2024-09-14T03:50:16.018 INFO:tasks.rados.rados.0.smithi110.stdout:3650: expect (ObjNum 1109 snap 353 seq_num 1109) 2024-09-14T03:50:16.062 INFO:tasks.rados.rados.0.smithi110.stdout:3651: finishing rollback tid 1 to smithi11075562-36 2024-09-14T03:50:16.062 INFO:tasks.rados.rados.0.smithi110.stdout:3651: finishing rollback tid 2 to smithi11075562-36 2024-09-14T03:50:16.062 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2519 (ObjNum 1076 snap 338 seq_num 1076) dirty exists 2024-09-14T03:50:16.062 INFO:tasks.rados.rados.0.smithi110.stdout:3656: finishing write tid 1 to smithi11075562-43 2024-09-14T03:50:16.062 INFO:tasks.rados.rados.0.smithi110.stdout:3656: finishing write tid 2 to smithi11075562-43 2024-09-14T03:50:16.062 INFO:tasks.rados.rados.0.smithi110.stdout:3656: finishing write tid 3 to smithi11075562-43 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3656: finishing write tid 4 to smithi11075562-43 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3656: finishing write tid 5 to smithi11075562-43 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3656: finishing write tid 6 to smithi11075562-43 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 2166 (ObjNum 1134 snap 367 seq_num 1134) dirty exists 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3656: left oid 43 (ObjNum 1134 snap 367 seq_num 1134) 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3652: finishing copy_from racing read to smithi11075562-16 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3650: done (5 left) 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3651: done (4 left) 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3653: done (3 left) 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3656: done (2 left) 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3661: rollback oid 14 current snap is 367 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 14 to 345 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:3652: finishing copy_from to smithi11075562-16 2024-09-14T03:50:16.063 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2148 (ObjNum 1124 snap 366 seq_num 1124) dirty exists 2024-09-14T03:50:16.064 INFO:tasks.rados.rados.0.smithi110.stdout:3660: finishing copy_from to smithi11075562-2 2024-09-14T03:50:16.064 INFO:tasks.rados.rados.0.smithi110.stdout:3660: got expected ENOENT (src dne) 2024-09-14T03:50:16.064 INFO:tasks.rados.rados.0.smithi110.stdout:3660: finishing copy_from racing read to smithi11075562-2 2024-09-14T03:50:16.064 INFO:tasks.rados.rados.0.smithi110.stdout:3652: done (2 left) 2024-09-14T03:50:16.064 INFO:tasks.rados.rados.0.smithi110.stdout:3660: done (1 left) 2024-09-14T03:50:16.064 INFO:tasks.rados.rados.0.smithi110.stdout:3662: delete oid 20 current snap is 367 2024-09-14T03:50:16.064 INFO:tasks.rados.rados.0.smithi110.stdout:3662: done (1 left) 2024-09-14T03:50:16.064 INFO:tasks.rados.rados.0.smithi110.stdout:3663: read oid 7 snap 364 2024-09-14T03:50:16.065 INFO:tasks.rados.rados.0.smithi110.stdout:3663: expect deleted 2024-09-14T03:50:16.065 INFO:tasks.rados.rados.0.smithi110.stdout:3664: setattr oid 19 current snap is 367 2024-09-14T03:50:16.065 INFO:tasks.rados.rados.0.smithi110.stdout:3661: finishing rollback tid 0 to smithi11075562-14 2024-09-14T03:50:16.066 INFO:tasks.rados.rados.0.smithi110.stdout:3661: finishing rollback tid 1 to smithi11075562-14 2024-09-14T03:50:16.066 INFO:tasks.rados.rados.0.smithi110.stdout:3661: finishing rollback tid 2 to smithi11075562-14 2024-09-14T03:50:16.066 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 2169 (ObjNum 1003 snap 323 seq_num 1003) dirty exists 2024-09-14T03:50:16.066 INFO:tasks.rados.rados.0.smithi110.stdout:3661: done (2 left) 2024-09-14T03:50:16.066 INFO:tasks.rados.rados.0.smithi110.stdout:3663: done (1 left) 2024-09-14T03:50:16.066 INFO:tasks.rados.rados.0.smithi110.stdout:3665: snap_create 2024-09-14T03:50:16.069 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2173 (ObjNum 1131 snap 367 seq_num 1131) dirty exists 2024-09-14T03:50:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:15 smithi110 ceph-mon[88262]: pgmap v838: 121 pgs: 121 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 485 KiB/s wr, 2 op/s 2024-09-14T03:50:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:15 smithi110 ceph-mon[88262]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T03:50:16.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:15 smithi110 ceph-mon[86423]: pgmap v838: 121 pgs: 121 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 485 KiB/s wr, 2 op/s 2024-09-14T03:50:16.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:15 smithi110 ceph-mon[86423]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T03:50:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:15 smithi195 ceph-mon[82388]: pgmap v838: 121 pgs: 121 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 485 KiB/s wr, 2 op/s 2024-09-14T03:50:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:15 smithi195 ceph-mon[82388]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T03:50:16.845 INFO:tasks.rados.rados.0.smithi110.stdout:3664: done (1 left) 2024-09-14T03:50:16.845 INFO:tasks.rados.rados.0.smithi110.stdout:3665: done (0 left) 2024-09-14T03:50:16.845 INFO:tasks.rados.rados.0.smithi110.stdout:3666: rmattr oid 41 current snap is 368 2024-09-14T03:50:16.845 INFO:tasks.rados.rados.0.smithi110.stdout:3667: read oid 24 snap -1 2024-09-14T03:50:16.845 INFO:tasks.rados.rados.0.smithi110.stdout:3667: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:50:16.845 INFO:tasks.rados.rados.0.smithi110.stdout:3668: copy_from oid 6 from oid 39 current snap is 368 2024-09-14T03:50:16.845 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 14 current snap is 368 2024-09-14T03:50:16.846 INFO:tasks.rados.rados.0.smithi110.stdout:3669: seq_num 1137 ranges {2408448=647168,3055616=696320,3751936=557056,4308992=352256} 2024-09-14T03:50:16.862 INFO:tasks.rados.rados.0.smithi110.stdout:3669: writing smithi11075562-14 from 2408448 to 3055616 tid 1 2024-09-14T03:50:16.866 INFO:tasks.rados.rados.0.smithi110.stdout:3669: writing smithi11075562-14 from 3055616 to 3751936 tid 2 2024-09-14T03:50:16.870 INFO:tasks.rados.rados.0.smithi110.stdout:3669: writing smithi11075562-14 from 3751936 to 4308992 tid 3 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:3669: writing smithi11075562-14 from 4308992 to 4661248 tid 4 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:3670: read oid 38 snap 363 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:3670: expect (ObjNum 1045 snap 330 seq_num 1045) 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:3671: snap_create 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:3668: finishing copy_from to smithi11075562-6 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:3668: got expected ENOENT (src dne) 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:3668: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 2170 (ObjNum 1008 snap 323 seq_num 1008) dirty exists 2024-09-14T03:50:16.873 INFO:tasks.rados.rados.0.smithi110.stdout:3669: finishing write tid 1 to smithi11075562-14 2024-09-14T03:50:16.874 INFO:tasks.rados.rados.0.smithi110.stdout:3669: finishing write tid 2 to smithi11075562-14 2024-09-14T03:50:16.879 INFO:tasks.rados.rados.0.smithi110.stdout:3669: finishing write tid 3 to smithi11075562-14 2024-09-14T03:50:16.879 INFO:tasks.rados.rados.0.smithi110.stdout:3669: finishing write tid 4 to smithi11075562-14 2024-09-14T03:50:16.879 INFO:tasks.rados.rados.0.smithi110.stdout:3669: finishing write tid 5 to smithi11075562-14 2024-09-14T03:50:16.881 INFO:tasks.rados.rados.0.smithi110.stdout:3669: finishing write tid 6 to smithi11075562-14 2024-09-14T03:50:16.881 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 14 v 2175 (ObjNum 1137 snap 368 seq_num 1137) dirty exists 2024-09-14T03:50:16.881 INFO:tasks.rados.rados.0.smithi110.stdout:3669: left oid 14 (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:50:16.884 INFO:tasks.rados.rados.0.smithi110.stdout:3670: expect (ObjNum 1045 snap 330 seq_num 1045) 2024-09-14T03:50:16.974 INFO:tasks.rados.rados.0.smithi110.stdout:3667: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:50:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:16 smithi110 ceph-mon[88262]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T03:50:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:16 smithi110 ceph-mon[88262]: pgmap v841: 121 pgs: 121 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-14T03:50:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:16 smithi110 ceph-mon[86423]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T03:50:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:16 smithi110 ceph-mon[86423]: pgmap v841: 121 pgs: 121 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-14T03:50:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:16 smithi195 ceph-mon[82388]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T03:50:17.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:16 smithi195 ceph-mon[82388]: pgmap v841: 121 pgs: 121 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-14T03:50:17.850 INFO:tasks.rados.rados.0.smithi110.stdout:3666: done (5 left) 2024-09-14T03:50:17.850 INFO:tasks.rados.rados.0.smithi110.stdout:3667: done (4 left) 2024-09-14T03:50:17.850 INFO:tasks.rados.rados.0.smithi110.stdout:3668: done (3 left) 2024-09-14T03:50:17.850 INFO:tasks.rados.rados.0.smithi110.stdout:3669: done (2 left) 2024-09-14T03:50:17.851 INFO:tasks.rados.rados.0.smithi110.stdout:3670: done (1 left) 2024-09-14T03:50:17.851 INFO:tasks.rados.rados.0.smithi110.stdout:3671: done (0 left) 2024-09-14T03:50:17.851 INFO:tasks.rados.rados.0.smithi110.stdout:3672: rmattr oid 27 current snap is 369 2024-09-14T03:50:17.851 INFO:tasks.rados.rados.0.smithi110.stdout:3672: done (0 left) 2024-09-14T03:50:17.851 INFO:tasks.rados.rados.0.smithi110.stdout:3673: copy_from oid 33 from oid 29 current snap is 369 2024-09-14T03:50:17.851 INFO:tasks.rados.rados.0.smithi110.stdout:3674: setattr oid 37 current snap is 369 2024-09-14T03:50:17.852 INFO:tasks.rados.rados.0.smithi110.stdout:3675: read oid 23 snap -1 2024-09-14T03:50:17.852 INFO:tasks.rados.rados.0.smithi110.stdout:3675: expect (ObjNum 1123 snap 366 seq_num 1123) 2024-09-14T03:50:17.852 INFO:tasks.rados.rados.0.smithi110.stdout:3676: rollback oid 27 current snap is 369 2024-09-14T03:50:17.852 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 27 to 345 2024-09-14T03:50:17.852 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 369 2024-09-14T03:50:17.852 INFO:tasks.rados.rados.0.smithi110.stdout:3677: seq_num 1138 ranges {3334144=663552,3997696=581632,4579328=712704,5292032=311296} 2024-09-14T03:50:17.873 INFO:tasks.rados.rados.0.smithi110.stdout:3677: writing smithi11075562-26 from 3334144 to 3997696 tid 1 2024-09-14T03:50:17.877 INFO:tasks.rados.rados.0.smithi110.stdout:3677: writing smithi11075562-26 from 3997696 to 4579328 tid 2 2024-09-14T03:50:17.882 INFO:tasks.rados.rados.0.smithi110.stdout:3677: writing smithi11075562-26 from 4579328 to 5292032 tid 3 2024-09-14T03:50:17.885 INFO:tasks.rados.rados.0.smithi110.stdout:3677: writing smithi11075562-26 from 5292032 to 5603328 tid 4 2024-09-14T03:50:17.885 INFO:tasks.rados.rados.0.smithi110.stdout:3678: rollback oid 45 current snap is 369 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 45 to 359 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3676: finishing rollback tid 1 to smithi11075562-27 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 0 (ObjNum 33 snap 0 seq_num 43181432) dirty dne 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3676: done (5 left) 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3679: setattr oid 42 current snap is 369 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 2520 (ObjNum 33 snap 0 seq_num 66283376) dirty exists 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3673: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3673: finishing copy_from to smithi11075562-33 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 1645 (ObjNum 1112 snap 353 seq_num 1112) dirty exists 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3677: finishing write tid 1 to smithi11075562-26 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3677: finishing write tid 2 to smithi11075562-26 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3673: done (5 left) 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3674: done (4 left) 2024-09-14T03:50:17.886 INFO:tasks.rados.rados.0.smithi110.stdout:3680: snap_create 2024-09-14T03:50:17.887 INFO:tasks.rados.rados.0.smithi110.stdout:3677: finishing write tid 3 to smithi11075562-26 2024-09-14T03:50:17.889 INFO:tasks.rados.rados.0.smithi110.stdout:3677: finishing write tid 4 to smithi11075562-26 2024-09-14T03:50:17.889 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2239 (ObjNum 1132 snap 367 seq_num 1132) dirty exists 2024-09-14T03:50:17.890 INFO:tasks.rados.rados.0.smithi110.stdout:3677: finishing write tid 5 to smithi11075562-26 2024-09-14T03:50:17.893 INFO:tasks.rados.rados.0.smithi110.stdout:3678: finishing rollback tid 0 to smithi11075562-45 2024-09-14T03:50:17.893 INFO:tasks.rados.rados.0.smithi110.stdout:3678: finishing rollback tid 1 to smithi11075562-45 2024-09-14T03:50:17.893 INFO:tasks.rados.rados.0.smithi110.stdout:3677: finishing write tid 6 to smithi11075562-26 2024-09-14T03:50:17.893 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 2009 (ObjNum 1138 snap 369 seq_num 1138) dirty exists 2024-09-14T03:50:17.894 INFO:tasks.rados.rados.0.smithi110.stdout:3677: left oid 26 (ObjNum 1138 snap 369 seq_num 1138) 2024-09-14T03:50:17.894 INFO:tasks.rados.rados.0.smithi110.stdout:3678: finishing rollback tid 2 to smithi11075562-45 2024-09-14T03:50:17.894 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2524 (ObjNum 1084 snap 343 seq_num 1084) dirty exists 2024-09-14T03:50:17.895 INFO:tasks.rados.rados.0.smithi110.stdout:3675: expect (ObjNum 1123 snap 366 seq_num 1123) 2024-09-14T03:50:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:17 smithi110 ceph-mon[88262]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T03:50:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:17 smithi110 ceph-mon[86423]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T03:50:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:17 smithi195 ceph-mon[82388]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T03:50:18.866 INFO:tasks.rados.rados.0.smithi110.stdout:3675: done (4 left) 2024-09-14T03:50:18.866 INFO:tasks.rados.rados.0.smithi110.stdout:3677: done (3 left) 2024-09-14T03:50:18.866 INFO:tasks.rados.rados.0.smithi110.stdout:3678: done (2 left) 2024-09-14T03:50:18.866 INFO:tasks.rados.rados.0.smithi110.stdout:3679: done (1 left) 2024-09-14T03:50:18.867 INFO:tasks.rados.rados.0.smithi110.stdout:3680: done (0 left) 2024-09-14T03:50:18.867 INFO:tasks.rados.rados.0.smithi110.stdout:3681: read oid 20 snap -1 2024-09-14T03:50:18.867 INFO:tasks.rados.rados.0.smithi110.stdout:3681: expect deleted 2024-09-14T03:50:18.867 INFO:tasks.rados.rados.0.smithi110.stdout:3682: snap_create 2024-09-14T03:50:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:18 smithi110 ceph-mon[88262]: pgmap v844: 121 pgs: 121 active+clean; 234 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-14T03:50:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:18 smithi110 ceph-mon[88262]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T03:50:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:18 smithi110 ceph-mon[86423]: pgmap v844: 121 pgs: 121 active+clean; 234 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-14T03:50:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:18 smithi110 ceph-mon[86423]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T03:50:19.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:18 smithi195 ceph-mon[82388]: pgmap v844: 121 pgs: 121 active+clean; 234 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-14T03:50:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:18 smithi195 ceph-mon[82388]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T03:50:19.869 INFO:tasks.rados.rados.0.smithi110.stdout:3681: done (1 left) 2024-09-14T03:50:19.869 INFO:tasks.rados.rados.0.smithi110.stdout:3682: done (0 left) 2024-09-14T03:50:19.869 INFO:tasks.rados.rados.0.smithi110.stdout:3683: snap_create 2024-09-14T03:50:20.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:19 smithi110 ceph-mon[86423]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T03:50:20.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:19 smithi110 ceph-mon[88262]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T03:50:20.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:19 smithi195 ceph-mon[82388]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T03:50:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:50:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:50:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:20.871 INFO:tasks.rados.rados.0.smithi110.stdout:3683: done (0 left) 2024-09-14T03:50:20.871 INFO:tasks.rados.rados.0.smithi110.stdout:3684: rmattr oid 8 current snap is 372 2024-09-14T03:50:20.871 INFO:tasks.rados.rados.0.smithi110.stdout:3685: copy_from oid 30 from oid 7 current snap is 372 2024-09-14T03:50:20.871 INFO:tasks.rados.rados.0.smithi110.stdout:3686: setattr oid 47 current snap is 372 2024-09-14T03:50:20.871 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 0 (ObjNum 1100 snap 349 seq_num 1100) dirty exists 2024-09-14T03:50:20.871 INFO:tasks.rados.rados.0.smithi110.stdout:3684: done (2 left) 2024-09-14T03:50:20.871 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 372 2024-09-14T03:50:20.872 INFO:tasks.rados.rados.0.smithi110.stdout:3687: seq_num 1139 ranges {1785856=679936,2465792=458752,2924544=524288,3448832=540672,3989504=81920} 2024-09-14T03:50:20.885 INFO:tasks.rados.rados.0.smithi110.stdout:3687: writing smithi11075562-25 from 1785856 to 2465792 tid 1 2024-09-14T03:50:20.887 INFO:tasks.rados.rados.0.smithi110.stdout:3687: writing smithi11075562-25 from 2465792 to 2924544 tid 2 2024-09-14T03:50:20.890 INFO:tasks.rados.rados.0.smithi110.stdout:3687: writing smithi11075562-25 from 2924544 to 3448832 tid 3 2024-09-14T03:50:20.893 INFO:tasks.rados.rados.0.smithi110.stdout:3687: writing smithi11075562-25 from 3448832 to 3989504 tid 4 2024-09-14T03:50:20.894 INFO:tasks.rados.rados.0.smithi110.stdout:3687: writing smithi11075562-25 from 3989504 to 4071424 tid 5 2024-09-14T03:50:20.894 INFO:tasks.rados.rados.0.smithi110.stdout:3688: snap_remove snap 345 2024-09-14T03:50:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:20 smithi110 ceph-mon[86423]: pgmap v846: 121 pgs: 121 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 6.0 MiB/s wr, 24 op/s 2024-09-14T03:50:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:20 smithi110 ceph-mon[86423]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T03:50:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:20 smithi110 ceph-mon[88262]: pgmap v846: 121 pgs: 121 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 6.0 MiB/s wr, 24 op/s 2024-09-14T03:50:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:20 smithi110 ceph-mon[88262]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T03:50:21.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:20 smithi195 ceph-mon[82388]: pgmap v846: 121 pgs: 121 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 6.0 MiB/s wr, 24 op/s 2024-09-14T03:50:21.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:20 smithi195 ceph-mon[82388]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T03:50:21.876 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:50:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:50:21] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.51.0" 2024-09-14T03:50:21.884 INFO:tasks.rados.rados.0.smithi110.stdout:3688: done (3 left) 2024-09-14T03:50:21.884 INFO:tasks.rados.rados.0.smithi110.stdout:3689: read oid 4 snap 359 2024-09-14T03:50:21.884 INFO:tasks.rados.rados.0.smithi110.stdout:3689: expect (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3690: rollback oid 16 current snap is 372 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 16 to 355 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3691: read oid 24 snap -1 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3691: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3692: rollback oid 49 current snap is 372 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 365 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3685: finishing copy_from to smithi11075562-30 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3685: got expected ENOENT (src dne) 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3685: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 47 v 2526 (ObjNum 1092 snap 343 seq_num 1092) dirty exists 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3687: finishing write tid 1 to smithi11075562-25 2024-09-14T03:50:21.885 INFO:tasks.rados.rados.0.smithi110.stdout:3685: done (6 left) 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3686: done (5 left) 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3693: rollback oid 21 current snap is 372 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 367 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3694: delete oid 13 current snap is 372 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3687: finishing write tid 2 to smithi11075562-25 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3687: finishing write tid 3 to smithi11075562-25 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3687: finishing write tid 4 to smithi11075562-25 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3687: finishing write tid 5 to smithi11075562-25 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3687: finishing write tid 6 to smithi11075562-25 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3687: finishing write tid 7 to smithi11075562-25 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2533 (ObjNum 1139 snap 372 seq_num 1139) dirty exists 2024-09-14T03:50:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3687: left oid 25 (ObjNum 1139 snap 372 seq_num 1139) 2024-09-14T03:50:21.897 INFO:tasks.rados.rados.0.smithi110.stdout:3690: finishing rollback tid 0 to smithi11075562-16 2024-09-14T03:50:21.900 INFO:tasks.rados.rados.0.smithi110.stdout:3692: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:50:21.900 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2210 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-14T03:50:21.900 INFO:tasks.rados.rados.0.smithi110.stdout:3687: done (6 left) 2024-09-14T03:50:21.900 INFO:tasks.rados.rados.0.smithi110.stdout:3692: done (5 left) 2024-09-14T03:50:21.900 INFO:tasks.rados.rados.0.smithi110.stdout:3694: done (4 left) 2024-09-14T03:50:21.900 INFO:tasks.rados.rados.0.smithi110.stdout:3695: delete oid 18 current snap is 372 2024-09-14T03:50:21.901 INFO:tasks.rados.rados.0.smithi110.stdout:3690: finishing rollback tid 1 to smithi11075562-16 2024-09-14T03:50:21.904 INFO:tasks.rados.rados.0.smithi110.stdout:3690: finishing rollback tid 2 to smithi11075562-16 2024-09-14T03:50:21.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2176 (ObjNum 1084 snap 343 seq_num 1084) dirty exists 2024-09-14T03:50:21.905 INFO:tasks.rados.rados.0.smithi110.stdout:3693: finishing rollback tid 0 to smithi11075562-21 2024-09-14T03:50:21.906 INFO:tasks.rados.rados.0.smithi110.stdout:3690: done (4 left) 2024-09-14T03:50:21.907 INFO:tasks.rados.rados.0.smithi110.stdout:3695: done (3 left) 2024-09-14T03:50:21.907 INFO:tasks.rados.rados.0.smithi110.stdout:3696: rmattr oid 38 current snap is 372 2024-09-14T03:50:21.907 INFO:tasks.rados.rados.0.smithi110.stdout:3696: done (3 left) 2024-09-14T03:50:21.907 INFO:tasks.rados.rados.0.smithi110.stdout:3697: read oid 20 snap -1 2024-09-14T03:50:21.907 INFO:tasks.rados.rados.0.smithi110.stdout:3697: expect deleted 2024-09-14T03:50:21.907 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 372 2024-09-14T03:50:21.907 INFO:tasks.rados.rados.0.smithi110.stdout:3698: seq_num 1142 ranges {0=729088,729088=565248,1294336=696320,1990656=344064} 2024-09-14T03:50:21.911 INFO:tasks.rados.rados.0.smithi110.stdout:3698: writing smithi11075562-27 from 0 to 729088 tid 1 2024-09-14T03:50:21.915 INFO:tasks.rados.rados.0.smithi110.stdout:3698: writing smithi11075562-27 from 729088 to 1294336 tid 2 2024-09-14T03:50:21.919 INFO:tasks.rados.rados.0.smithi110.stdout:3698: writing smithi11075562-27 from 1294336 to 1990656 tid 3 2024-09-14T03:50:21.922 INFO:tasks.rados.rados.0.smithi110.stdout:3698: writing smithi11075562-27 from 1990656 to 2334720 tid 4 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3699: copy_from oid 6 from oid 18 current snap is 372 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3693: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3693: finishing rollback tid 2 to smithi11075562-21 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2178 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3698: finishing write tid 1 to smithi11075562-27 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3693: done (5 left) 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3697: done (4 left) 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3700: read oid 28 snap -1 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3700: expect deleted 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3701: read oid 35 snap -1 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3701: expect (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3702: copy_from oid 42 from oid 38 current snap is 372 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3698: finishing write tid 2 to smithi11075562-27 2024-09-14T03:50:21.923 INFO:tasks.rados.rados.0.smithi110.stdout:3703: delete oid 49 current snap is 372 2024-09-14T03:50:21.924 INFO:tasks.rados.rados.0.smithi110.stdout:3699: finishing copy_from to smithi11075562-6 2024-09-14T03:50:21.924 INFO:tasks.rados.rados.0.smithi110.stdout:3699: got expected ENOENT (src dne) 2024-09-14T03:50:21.924 INFO:tasks.rados.rados.0.smithi110.stdout:3699: finishing copy_from racing read to smithi11075562-6 2024-09-14T03:50:21.925 INFO:tasks.rados.rados.0.smithi110.stdout:3698: finishing write tid 3 to smithi11075562-27 2024-09-14T03:50:21.925 INFO:tasks.rados.rados.0.smithi110.stdout:3698: finishing write tid 4 to smithi11075562-27 2024-09-14T03:50:21.926 INFO:tasks.rados.rados.0.smithi110.stdout:3698: finishing write tid 5 to smithi11075562-27 2024-09-14T03:50:21.928 INFO:tasks.rados.rados.0.smithi110.stdout:3698: finishing write tid 6 to smithi11075562-27 2024-09-14T03:50:21.928 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1817 (ObjNum 1142 snap 372 seq_num 1142) dirty exists 2024-09-14T03:50:21.929 INFO:tasks.rados.rados.0.smithi110.stdout:3698: left oid 27 (ObjNum 1142 snap 372 seq_num 1142) 2024-09-14T03:50:21.930 INFO:tasks.rados.rados.0.smithi110.stdout:3701: expect (ObjNum 1060 snap 335 seq_num 1060) 2024-09-14T03:50:21.960 INFO:tasks.rados.rados.0.smithi110.stdout:3702: finishing copy_from racing read to smithi11075562-42 2024-09-14T03:50:21.960 INFO:tasks.rados.rados.0.smithi110.stdout:3702: finishing copy_from to smithi11075562-42 2024-09-14T03:50:21.960 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 1651 (ObjNum 1127 snap 366 seq_num 1127) dirty exists 2024-09-14T03:50:21.960 INFO:tasks.rados.rados.0.smithi110.stdout:3691: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:50:22.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:21 smithi110 ceph-mon[88262]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T03:50:22.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:21 smithi110 ceph-mon[86423]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T03:50:22.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:50:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:50:21.968+0000 7fba3414e640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1a8 after trimming was started 2024-09-14T03:50:22.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:21 smithi195 ceph-mon[82388]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T03:50:22.671 INFO:tasks.rados.rados.0.smithi110.stdout:3689: expect (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:50:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:22 smithi110 ceph-mon[88262]: pgmap v849: 121 pgs: 121 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T03:50:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:22 smithi110 ceph-mon[88262]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T03:50:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:22 smithi110 ceph-mon[86423]: pgmap v849: 121 pgs: 121 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T03:50:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:22 smithi110 ceph-mon[86423]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T03:50:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:22 smithi195 ceph-mon[82388]: pgmap v849: 121 pgs: 121 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T03:50:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:22 smithi195 ceph-mon[82388]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T03:50:23.821 INFO:tasks.rados.rados.0.smithi110.stdout:3689: done (7 left) 2024-09-14T03:50:23.821 INFO:tasks.rados.rados.0.smithi110.stdout:3691: done (6 left) 2024-09-14T03:50:23.821 INFO:tasks.rados.rados.0.smithi110.stdout:3698: done (5 left) 2024-09-14T03:50:23.822 INFO:tasks.rados.rados.0.smithi110.stdout:3699: done (4 left) 2024-09-14T03:50:23.822 INFO:tasks.rados.rados.0.smithi110.stdout:3700: done (3 left) 2024-09-14T03:50:23.822 INFO:tasks.rados.rados.0.smithi110.stdout:3701: done (2 left) 2024-09-14T03:50:23.822 INFO:tasks.rados.rados.0.smithi110.stdout:3702: done (1 left) 2024-09-14T03:50:23.822 INFO:tasks.rados.rados.0.smithi110.stdout:3703: done (0 left) 2024-09-14T03:50:23.822 INFO:tasks.rados.rados.0.smithi110.stdout:3704: snap_create 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3704: done (0 left) 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3705: read oid 47 snap 349 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3705: expect (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3706: copy_from oid 43 from oid 38 current snap is 373 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3707: read oid 2 snap -1 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3707: expect (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3708: read oid 10 snap 371 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3708: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 373 2024-09-14T03:50:23.829 INFO:tasks.rados.rados.0.smithi110.stdout:3709: seq_num 1144 ranges {4530176=770048,5300224=737280,6037504=712704,6750208=155648} 2024-09-14T03:50:23.860 INFO:tasks.rados.rados.0.smithi110.stdout:3709: writing smithi11075562-42 from 4530176 to 5300224 tid 1 2024-09-14T03:50:23.865 INFO:tasks.rados.rados.0.smithi110.stdout:3709: writing smithi11075562-42 from 5300224 to 6037504 tid 2 2024-09-14T03:50:23.869 INFO:tasks.rados.rados.0.smithi110.stdout:3709: writing smithi11075562-42 from 6037504 to 6750208 tid 3 2024-09-14T03:50:23.870 INFO:tasks.rados.rados.0.smithi110.stdout:3709: writing smithi11075562-42 from 6750208 to 6905856 tid 4 2024-09-14T03:50:23.870 INFO:tasks.rados.rados.0.smithi110.stdout:3710: rollback oid 15 current snap is 373 2024-09-14T03:50:23.870 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 15 to 352 2024-09-14T03:50:23.870 INFO:tasks.rados.rados.0.smithi110.stdout:3707: expect (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:50:23.899 INFO:tasks.rados.rados.0.smithi110.stdout:3705: expect (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:50:23.939 INFO:tasks.rados.rados.0.smithi110.stdout:3706: finishing copy_from racing read to smithi11075562-43 2024-09-14T03:50:23.939 INFO:tasks.rados.rados.0.smithi110.stdout:3706: finishing copy_from to smithi11075562-43 2024-09-14T03:50:23.939 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 1651 (ObjNum 1127 snap 366 seq_num 1127) dirty exists 2024-09-14T03:50:23.939 INFO:tasks.rados.rados.0.smithi110.stdout:3708: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:50:24.379 INFO:tasks.rados.rados.0.smithi110.stdout:3709: finishing write tid 1 to smithi11075562-42 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3709: finishing write tid 2 to smithi11075562-42 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3710: finishing rollback tid 1 to smithi11075562-15 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3709: finishing write tid 3 to smithi11075562-42 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3710: finishing rollback tid 2 to smithi11075562-15 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2505 (ObjNum 1055 snap 332 seq_num 1055) dirty exists 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3709: finishing write tid 4 to smithi11075562-42 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3705: done (5 left) 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3706: done (4 left) 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3707: done (3 left) 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3708: done (2 left) 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3710: done (1 left) 2024-09-14T03:50:24.380 INFO:tasks.rados.rados.0.smithi110.stdout:3711: read oid 43 snap -1 2024-09-14T03:50:24.381 INFO:tasks.rados.rados.0.smithi110.stdout:3711: expect (ObjNum 1127 snap 366 seq_num 1127) 2024-09-14T03:50:24.381 INFO:tasks.rados.rados.0.smithi110.stdout:3712: snap_create 2024-09-14T03:50:24.381 INFO:tasks.rados.rados.0.smithi110.stdout:3709: finishing write tid 5 to smithi11075562-42 2024-09-14T03:50:24.381 INFO:tasks.rados.rados.0.smithi110.stdout:3709: finishing write tid 6 to smithi11075562-42 2024-09-14T03:50:24.381 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2255 (ObjNum 1144 snap 373 seq_num 1144) dirty exists 2024-09-14T03:50:24.381 INFO:tasks.rados.rados.0.smithi110.stdout:3709: left oid 42 (ObjNum 1144 snap 373 seq_num 1144) 2024-09-14T03:50:24.393 INFO:tasks.rados.rados.0.smithi110.stdout:3711: expect (ObjNum 1127 snap 366 seq_num 1127) 2024-09-14T03:50:24.833 INFO:tasks.rados.rados.0.smithi110.stdout:3709: done (2 left) 2024-09-14T03:50:24.833 INFO:tasks.rados.rados.0.smithi110.stdout:3711: done (1 left) 2024-09-14T03:50:24.833 INFO:tasks.rados.rados.0.smithi110.stdout:3712: done (0 left) 2024-09-14T03:50:24.833 INFO:tasks.rados.rados.0.smithi110.stdout:3713: read oid 49 snap 370 2024-09-14T03:50:24.833 INFO:tasks.rados.rados.0.smithi110.stdout:3713: expect deleted 2024-09-14T03:50:24.834 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 374 2024-09-14T03:50:24.834 INFO:tasks.rados.rados.0.smithi110.stdout:3714: seq_num 1145 ranges {5603328=786432,6389760=622592,7012352=516096,7528448=417792,7946240=49152} 2024-09-14T03:50:24.866 INFO:tasks.rados.rados.0.smithi110.stdout:3714: writing smithi11075562-26 from 5603328 to 6389760 tid 1 2024-09-14T03:50:24.870 INFO:tasks.rados.rados.0.smithi110.stdout:3714: writing smithi11075562-26 from 6389760 to 7012352 tid 2 2024-09-14T03:50:24.873 INFO:tasks.rados.rados.0.smithi110.stdout:3714: writing smithi11075562-26 from 7012352 to 7528448 tid 3 2024-09-14T03:50:24.875 INFO:tasks.rados.rados.0.smithi110.stdout:3714: writing smithi11075562-26 from 7528448 to 7946240 tid 4 2024-09-14T03:50:24.876 INFO:tasks.rados.rados.0.smithi110.stdout:3714: writing smithi11075562-26 from 7946240 to 7995392 tid 5 2024-09-14T03:50:24.876 INFO:tasks.rados.rados.0.smithi110.stdout:3715: rollback oid 34 current snap is 374 2024-09-14T03:50:24.876 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 359 2024-09-14T03:50:24.876 INFO:tasks.rados.rados.0.smithi110.stdout:3714: finishing write tid 1 to smithi11075562-26 2024-09-14T03:50:24.876 INFO:tasks.rados.rados.0.smithi110.stdout:3714: finishing write tid 2 to smithi11075562-26 2024-09-14T03:50:24.876 INFO:tasks.rados.rados.0.smithi110.stdout:3713: done (2 left) 2024-09-14T03:50:24.876 INFO:tasks.rados.rados.0.smithi110.stdout:3716: delete oid 8 current snap is 374 2024-09-14T03:50:24.877 INFO:tasks.rados.rados.0.smithi110.stdout:3714: finishing write tid 3 to smithi11075562-26 2024-09-14T03:50:24.878 INFO:tasks.rados.rados.0.smithi110.stdout:3714: finishing write tid 4 to smithi11075562-26 2024-09-14T03:50:24.879 INFO:tasks.rados.rados.0.smithi110.stdout:3714: finishing write tid 5 to smithi11075562-26 2024-09-14T03:50:24.879 INFO:tasks.rados.rados.0.smithi110.stdout:3714: finishing write tid 6 to smithi11075562-26 2024-09-14T03:50:24.880 INFO:tasks.rados.rados.0.smithi110.stdout:3714: finishing write tid 7 to smithi11075562-26 2024-09-14T03:50:24.880 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 2015 (ObjNum 1145 snap 374 seq_num 1145) dirty exists 2024-09-14T03:50:24.880 INFO:tasks.rados.rados.0.smithi110.stdout:3714: left oid 26 (ObjNum 1145 snap 374 seq_num 1145) 2024-09-14T03:50:24.881 INFO:tasks.rados.rados.0.smithi110.stdout:3714: done (2 left) 2024-09-14T03:50:24.881 INFO:tasks.rados.rados.0.smithi110.stdout:3716: done (1 left) 2024-09-14T03:50:24.881 INFO:tasks.rados.rados.0.smithi110.stdout:3717: read oid 2 snap -1 2024-09-14T03:50:24.881 INFO:tasks.rados.rados.0.smithi110.stdout:3717: expect (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:50:24.881 INFO:tasks.rados.rados.0.smithi110.stdout:3718: copy_from oid 50 from oid 10 current snap is 374 2024-09-14T03:50:24.881 INFO:tasks.rados.rados.0.smithi110.stdout:3719: snap_remove snap 364 2024-09-14T03:50:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:24 smithi110 ceph-mon[88262]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T03:50:25.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:24 smithi110 ceph-mon[86423]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T03:50:25.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:24 smithi195 ceph-mon[82388]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T03:50:25.839 INFO:tasks.rados.rados.0.smithi110.stdout:3719: done (3 left) 2024-09-14T03:50:25.839 INFO:tasks.rados.rados.0.smithi110.stdout:3720: snap_remove snap 355 2024-09-14T03:50:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:25 smithi110 ceph-mon[86423]: pgmap v852: 121 pgs: 121 active+clean; 230 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 554 KiB/s wr, 15 op/s 2024-09-14T03:50:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:25 smithi110 ceph-mon[86423]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T03:50:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:25 smithi110 ceph-mon[88262]: pgmap v852: 121 pgs: 121 active+clean; 230 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 554 KiB/s wr, 15 op/s 2024-09-14T03:50:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:25 smithi110 ceph-mon[88262]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T03:50:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:25 smithi195 ceph-mon[82388]: pgmap v852: 121 pgs: 121 active+clean; 230 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 554 KiB/s wr, 15 op/s 2024-09-14T03:50:26.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:25 smithi195 ceph-mon[82388]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T03:50:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:3720: done (3 left) 2024-09-14T03:50:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:3721: rollback oid 24 current snap is 374 2024-09-14T03:50:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 349 2024-09-14T03:50:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:3715: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:50:26.845 INFO:tasks.rados.rados.0.smithi110.stdout:3715: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:50:26.846 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 2168 (ObjNum 65 snap 0 seq_num 65515536) dirty dne 2024-09-14T03:50:26.846 INFO:tasks.rados.rados.0.smithi110.stdout:3717: expect (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3718: finishing copy_from racing read to smithi11075562-50 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3718: finishing copy_from to smithi11075562-50 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 50 v 2047 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3721: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3721: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3721: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2050 (ObjNum 1082 snap 342 seq_num 1082) dirty exists 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3715: done (3 left) 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3717: done (2 left) 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3718: done (1 left) 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3721: done (0 left) 2024-09-14T03:50:26.883 INFO:tasks.rados.rados.0.smithi110.stdout:3722: delete oid 32 current snap is 374 2024-09-14T03:50:26.885 INFO:tasks.rados.rados.0.smithi110.stdout:3722: done (0 left) 2024-09-14T03:50:26.885 INFO:tasks.rados.rados.0.smithi110.stdout:3723: snap_create 2024-09-14T03:50:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:26 smithi110 ceph-mon[86423]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T03:50:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:26 smithi110 ceph-mon[86423]: pgmap v855: 121 pgs: 121 active+clean; 226 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-14T03:50:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:26 smithi110 ceph-mon[88262]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T03:50:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:26 smithi110 ceph-mon[88262]: pgmap v855: 121 pgs: 121 active+clean; 226 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-14T03:50:27.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:26 smithi195 ceph-mon[82388]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T03:50:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:26 smithi195 ceph-mon[82388]: pgmap v855: 121 pgs: 121 active+clean; 226 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-14T03:50:27.850 INFO:tasks.rados.rados.0.smithi110.stdout:3723: done (0 left) 2024-09-14T03:50:27.850 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 31 current snap is 375 2024-09-14T03:50:27.850 INFO:tasks.rados.rados.0.smithi110.stdout:3724: seq_num 1148 ranges {0=32768} 2024-09-14T03:50:27.850 INFO:tasks.rados.rados.0.smithi110.stdout:3724: writing smithi11075562-31 from 0 to 32768 tid 1 2024-09-14T03:50:27.850 INFO:tasks.rados.rados.0.smithi110.stdout:3725: delete oid 30 current snap is 375 2024-09-14T03:50:27.855 INFO:tasks.rados.rados.0.smithi110.stdout:3724: finishing write tid 1 to smithi11075562-31 2024-09-14T03:50:27.855 INFO:tasks.rados.rados.0.smithi110.stdout:3724: finishing write tid 2 to smithi11075562-31 2024-09-14T03:50:27.856 INFO:tasks.rados.rados.0.smithi110.stdout:3724: finishing write tid 3 to smithi11075562-31 2024-09-14T03:50:27.856 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 2170 (ObjNum 1148 snap 375 seq_num 1148) dirty exists 2024-09-14T03:50:27.856 INFO:tasks.rados.rados.0.smithi110.stdout:3724: left oid 31 (ObjNum 1148 snap 375 seq_num 1148) 2024-09-14T03:50:27.857 INFO:tasks.rados.rados.0.smithi110.stdout:3724: done (1 left) 2024-09-14T03:50:27.857 INFO:tasks.rados.rados.0.smithi110.stdout:3725: done (0 left) 2024-09-14T03:50:27.857 INFO:tasks.rados.rados.0.smithi110.stdout:3726: delete oid 48 current snap is 375 2024-09-14T03:50:27.859 INFO:tasks.rados.rados.0.smithi110.stdout:3726: done (0 left) 2024-09-14T03:50:27.859 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 44 current snap is 375 2024-09-14T03:50:27.859 INFO:tasks.rados.rados.0.smithi110.stdout:3727: seq_num 1151 ranges {9682944=81920} 2024-09-14T03:50:27.911 INFO:tasks.rados.rados.0.smithi110.stdout:3727: writing smithi11075562-44 from 9682944 to 9764864 tid 1 2024-09-14T03:50:27.911 INFO:tasks.rados.rados.0.smithi110.stdout:3728: snap_remove snap 356 2024-09-14T03:50:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:27 smithi110 ceph-mon[88262]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T03:50:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:27 smithi110 ceph-mon[86423]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T03:50:28.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:28.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:27 smithi195 ceph-mon[82388]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T03:50:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:28.856 INFO:tasks.rados.rados.0.smithi110.stdout:3727: finishing write tid 1 to smithi11075562-44 2024-09-14T03:50:28.856 INFO:tasks.rados.rados.0.smithi110.stdout:3727: finishing write tid 2 to smithi11075562-44 2024-09-14T03:50:28.857 INFO:tasks.rados.rados.0.smithi110.stdout:3727: finishing write tid 3 to smithi11075562-44 2024-09-14T03:50:28.857 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 2149 (ObjNum 1151 snap 375 seq_num 1151) dirty exists 2024-09-14T03:50:28.857 INFO:tasks.rados.rados.0.smithi110.stdout:3727: left oid 44 (ObjNum 1151 snap 375 seq_num 1151) 2024-09-14T03:50:28.857 INFO:tasks.rados.rados.0.smithi110.stdout:3727: done (1 left) 2024-09-14T03:50:28.857 INFO:tasks.rados.rados.0.smithi110.stdout:3728: done (0 left) 2024-09-14T03:50:28.857 INFO:tasks.rados.rados.0.smithi110.stdout:3729: snap_create 2024-09-14T03:50:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:28 smithi110 ceph-mon[88262]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T03:50:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:28 smithi110 ceph-mon[88262]: pgmap v858: 121 pgs: 121 active+clean; 238 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 4.4 MiB/s wr, 21 op/s 2024-09-14T03:50:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:28 smithi110 ceph-mon[86423]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T03:50:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:28 smithi110 ceph-mon[86423]: pgmap v858: 121 pgs: 121 active+clean; 238 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 4.4 MiB/s wr, 21 op/s 2024-09-14T03:50:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:28 smithi195 ceph-mon[82388]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T03:50:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:28 smithi195 ceph-mon[82388]: pgmap v858: 121 pgs: 121 active+clean; 238 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 4.4 MiB/s wr, 21 op/s 2024-09-14T03:50:29.856 INFO:tasks.rados.rados.0.smithi110.stdout:3729: done (0 left) 2024-09-14T03:50:29.856 INFO:tasks.rados.rados.0.smithi110.stdout:3730: rollback oid 21 current snap is 376 2024-09-14T03:50:29.856 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 372 2024-09-14T03:50:29.856 INFO:tasks.rados.rados.0.smithi110.stdout:3731: rollback oid 38 current snap is 376 2024-09-14T03:50:29.856 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 371 2024-09-14T03:50:29.856 INFO:tasks.rados.rados.0.smithi110.stdout:3732: snap_create 2024-09-14T03:50:29.862 INFO:tasks.rados.rados.0.smithi110.stdout:3730: finishing rollback tid 0 to smithi11075562-21 2024-09-14T03:50:29.864 INFO:tasks.rados.rados.0.smithi110.stdout:3730: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:50:29.864 INFO:tasks.rados.rados.0.smithi110.stdout:3730: finishing rollback tid 2 to smithi11075562-21 2024-09-14T03:50:29.864 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2181 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:50:29.867 INFO:tasks.rados.rados.0.smithi110.stdout:3731: finishing rollback tid 0 to smithi11075562-38 2024-09-14T03:50:29.867 INFO:tasks.rados.rados.0.smithi110.stdout:3731: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:50:29.867 INFO:tasks.rados.rados.0.smithi110.stdout:3731: finishing rollback tid 2 to smithi11075562-38 2024-09-14T03:50:29.867 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1654 (ObjNum 1127 snap 366 seq_num 1127) dirty exists 2024-09-14T03:50:30.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:29 smithi110 ceph-mon[88262]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T03:50:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:29 smithi110 ceph-mon[86423]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T03:50:30.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:29 smithi195 ceph-mon[82388]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T03:50:30.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:50:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:50:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:30.867 INFO:tasks.rados.rados.0.smithi110.stdout:3730: done (2 left) 2024-09-14T03:50:30.868 INFO:tasks.rados.rados.0.smithi110.stdout:3731: done (1 left) 2024-09-14T03:50:30.868 INFO:tasks.rados.rados.0.smithi110.stdout:3732: done (0 left) 2024-09-14T03:50:30.868 INFO:tasks.rados.rados.0.smithi110.stdout:3733: read oid 9 snap 374 2024-09-14T03:50:30.868 INFO:tasks.rados.rados.0.smithi110.stdout:3733: expect deleted 2024-09-14T03:50:30.868 INFO:tasks.rados.rados.0.smithi110.stdout:3734: delete oid 37 current snap is 377 2024-09-14T03:50:30.875 INFO:tasks.rados.rados.0.smithi110.stdout:3733: done (1 left) 2024-09-14T03:50:30.875 INFO:tasks.rados.rados.0.smithi110.stdout:3734: done (0 left) 2024-09-14T03:50:30.875 INFO:tasks.rados.rados.0.smithi110.stdout:3735: rollback oid 10 current snap is 377 2024-09-14T03:50:30.875 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 10 to 363 2024-09-14T03:50:30.875 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 377 2024-09-14T03:50:30.875 INFO:tasks.rados.rados.0.smithi110.stdout:3736: seq_num 1153 ranges {4956160=114688} 2024-09-14T03:50:30.902 INFO:tasks.rados.rados.0.smithi110.stdout:3736: writing smithi11075562-1 from 4956160 to 5070848 tid 1 2024-09-14T03:50:30.902 INFO:tasks.rados.rados.0.smithi110.stdout:3737: rmattr oid 15 current snap is 377 2024-09-14T03:50:30.902 INFO:tasks.rados.rados.0.smithi110.stdout:3735: finishing rollback tid 0 to smithi11075562-10 2024-09-14T03:50:30.902 INFO:tasks.rados.rados.0.smithi110.stdout:3735: finishing rollback tid 1 to smithi11075562-10 2024-09-14T03:50:30.902 INFO:tasks.rados.rados.0.smithi110.stdout:3738: copy_from oid 8 from oid 36 current snap is 377 2024-09-14T03:50:30.903 INFO:tasks.rados.rados.0.smithi110.stdout:3735: finishing rollback tid 2 to smithi11075562-10 2024-09-14T03:50:30.903 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2053 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:50:30.903 INFO:tasks.rados.rados.0.smithi110.stdout:3735: done (3 left) 2024-09-14T03:50:30.903 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 377 2024-09-14T03:50:30.903 INFO:tasks.rados.rados.0.smithi110.stdout:3739: seq_num 1154 ranges {4071424=131072} 2024-09-14T03:50:30.925 INFO:tasks.rados.rados.0.smithi110.stdout:3739: writing smithi11075562-25 from 4071424 to 4202496 tid 1 2024-09-14T03:50:30.925 INFO:tasks.rados.rados.0.smithi110.stdout:3740: delete oid 16 current snap is 377 2024-09-14T03:50:30.925 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2508 (ObjNum 1055 snap 332 seq_num 1055) dirty exists 2024-09-14T03:50:30.926 INFO:tasks.rados.rados.0.smithi110.stdout:3736: finishing write tid 1 to smithi11075562-1 2024-09-14T03:50:30.926 INFO:tasks.rados.rados.0.smithi110.stdout:3736: finishing write tid 2 to smithi11075562-1 2024-09-14T03:50:30.926 INFO:tasks.rados.rados.0.smithi110.stdout:3736: finishing write tid 3 to smithi11075562-1 2024-09-14T03:50:30.926 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2264 (ObjNum 1153 snap 377 seq_num 1153) dirty exists 2024-09-14T03:50:30.926 INFO:tasks.rados.rados.0.smithi110.stdout:3736: left oid 1 (ObjNum 1153 snap 377 seq_num 1153) 2024-09-14T03:50:30.929 INFO:tasks.rados.rados.0.smithi110.stdout:3736: done (4 left) 2024-09-14T03:50:30.929 INFO:tasks.rados.rados.0.smithi110.stdout:3737: done (3 left) 2024-09-14T03:50:30.929 INFO:tasks.rados.rados.0.smithi110.stdout:3740: done (2 left) 2024-09-14T03:50:30.929 INFO:tasks.rados.rados.0.smithi110.stdout:3741: rollback oid 37 current snap is 377 2024-09-14T03:50:30.929 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 37 to 362 2024-09-14T03:50:30.929 INFO:tasks.rados.rados.0.smithi110.stdout:3742: setattr oid 31 current snap is 377 2024-09-14T03:50:30.932 INFO:tasks.rados.rados.0.smithi110.stdout:3743: read oid 27 snap -1 2024-09-14T03:50:30.932 INFO:tasks.rados.rados.0.smithi110.stdout:3743: expect (ObjNum 1142 snap 372 seq_num 1142) 2024-09-14T03:50:30.932 INFO:tasks.rados.rados.0.smithi110.stdout:3744: read oid 14 snap -1 2024-09-14T03:50:30.932 INFO:tasks.rados.rados.0.smithi110.stdout:3744: expect (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:50:30.932 INFO:tasks.rados.rados.0.smithi110.stdout:3745: read oid 22 snap -1 2024-09-14T03:50:30.932 INFO:tasks.rados.rados.0.smithi110.stdout:3745: expect (ObjNum 1102 snap 349 seq_num 1102) 2024-09-14T03:50:30.932 INFO:tasks.rados.rados.0.smithi110.stdout:3746: read oid 7 snap -1 2024-09-14T03:50:30.932 INFO:tasks.rados.rados.0.smithi110.stdout:3746: expect deleted 2024-09-14T03:50:30.933 INFO:tasks.rados.rados.0.smithi110.stdout:3747: delete oid 45 current snap is 377 2024-09-14T03:50:30.933 INFO:tasks.rados.rados.0.smithi110.stdout:3739: finishing write tid 1 to smithi11075562-25 2024-09-14T03:50:30.933 INFO:tasks.rados.rados.0.smithi110.stdout:3739: finishing write tid 2 to smithi11075562-25 2024-09-14T03:50:30.933 INFO:tasks.rados.rados.0.smithi110.stdout:3738: finishing copy_from racing read to smithi11075562-8 2024-09-14T03:50:30.937 INFO:tasks.rados.rados.0.smithi110.stdout:3741: finishing rollback tid 1 to smithi11075562-37 2024-09-14T03:50:30.937 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 2575 (ObjNum 33 snap 0 seq_num 65151760) dirty dne 2024-09-14T03:50:30.937 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 2171 (ObjNum 1148 snap 375 seq_num 1148) dirty exists 2024-09-14T03:50:30.938 INFO:tasks.rados.rados.0.smithi110.stdout:3739: finishing write tid 3 to smithi11075562-25 2024-09-14T03:50:30.938 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2578 (ObjNum 1154 snap 377 seq_num 1154) dirty exists 2024-09-14T03:50:30.938 INFO:tasks.rados.rados.0.smithi110.stdout:3739: left oid 25 (ObjNum 1154 snap 377 seq_num 1154) 2024-09-14T03:50:30.940 INFO:tasks.rados.rados.0.smithi110.stdout:3739: done (8 left) 2024-09-14T03:50:30.940 INFO:tasks.rados.rados.0.smithi110.stdout:3741: done (7 left) 2024-09-14T03:50:30.940 INFO:tasks.rados.rados.0.smithi110.stdout:3742: done (6 left) 2024-09-14T03:50:30.940 INFO:tasks.rados.rados.0.smithi110.stdout:3746: done (5 left) 2024-09-14T03:50:30.940 INFO:tasks.rados.rados.0.smithi110.stdout:3747: done (4 left) 2024-09-14T03:50:30.940 INFO:tasks.rados.rados.0.smithi110.stdout:3748: rollback oid 2 current snap is 377 2024-09-14T03:50:30.940 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 2 to 359 2024-09-14T03:50:30.940 INFO:tasks.rados.rados.0.smithi110.stdout:3749: snap_create 2024-09-14T03:50:30.944 INFO:tasks.rados.rados.0.smithi110.stdout:3743: expect (ObjNum 1142 snap 372 seq_num 1142) 2024-09-14T03:50:31.005 INFO:tasks.rados.rados.0.smithi110.stdout:3745: expect (ObjNum 1102 snap 349 seq_num 1102) 2024-09-14T03:50:31.048 INFO:tasks.rados.rados.0.smithi110.stdout:3738: finishing copy_from to smithi11075562-8 2024-09-14T03:50:31.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 2519 (ObjNum 1076 snap 338 seq_num 1076) dirty exists 2024-09-14T03:50:31.049 INFO:tasks.rados.rados.0.smithi110.stdout:3748: finishing rollback tid 0 to smithi11075562-2 2024-09-14T03:50:31.049 INFO:tasks.rados.rados.0.smithi110.stdout:3748: finishing rollback tid 1 to smithi11075562-2 2024-09-14T03:50:31.049 INFO:tasks.rados.rados.0.smithi110.stdout:3748: finishing rollback tid 2 to smithi11075562-2 2024-09-14T03:50:31.049 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 2151 (ObjNum 1066 snap 336 seq_num 1066) dirty exists 2024-09-14T03:50:31.049 INFO:tasks.rados.rados.0.smithi110.stdout:3744: expect (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:50:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:30 smithi110 ceph-mon[86423]: pgmap v860: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 4.3 MiB/s wr, 9 op/s 2024-09-14T03:50:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:30 smithi110 ceph-mon[86423]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T03:50:31.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:30 smithi110 ceph-mon[88262]: pgmap v860: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 4.3 MiB/s wr, 9 op/s 2024-09-14T03:50:31.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:30 smithi110 ceph-mon[88262]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T03:50:31.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:30 smithi195 ceph-mon[82388]: pgmap v860: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 4.3 MiB/s wr, 9 op/s 2024-09-14T03:50:31.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:30 smithi195 ceph-mon[82388]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T03:50:31.868 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:50:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:50:31] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-14T03:50:31.872 INFO:tasks.rados.rados.0.smithi110.stdout:3738: done (5 left) 2024-09-14T03:50:31.872 INFO:tasks.rados.rados.0.smithi110.stdout:3743: done (4 left) 2024-09-14T03:50:31.872 INFO:tasks.rados.rados.0.smithi110.stdout:3744: done (3 left) 2024-09-14T03:50:31.873 INFO:tasks.rados.rados.0.smithi110.stdout:3745: done (2 left) 2024-09-14T03:50:31.873 INFO:tasks.rados.rados.0.smithi110.stdout:3748: done (1 left) 2024-09-14T03:50:31.873 INFO:tasks.rados.rados.0.smithi110.stdout:3749: done (0 left) 2024-09-14T03:50:31.873 INFO:tasks.rados.rados.0.smithi110.stdout:3750: copy_from oid 10 from oid 7 current snap is 378 2024-09-14T03:50:31.873 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 35 current snap is 378 2024-09-14T03:50:31.873 INFO:tasks.rados.rados.0.smithi110.stdout:3751: seq_num 1157 ranges {958464=180224} 2024-09-14T03:50:31.878 INFO:tasks.rados.rados.0.smithi110.stdout:3751: writing smithi11075562-35 from 958464 to 1138688 tid 1 2024-09-14T03:50:31.879 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 378 2024-09-14T03:50:31.879 INFO:tasks.rados.rados.0.smithi110.stdout:3752: seq_num 1158 ranges {1449984=196608} 2024-09-14T03:50:31.888 INFO:tasks.rados.rados.0.smithi110.stdout:3752: writing smithi11075562-15 from 1449984 to 1646592 tid 1 2024-09-14T03:50:31.888 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 5 current snap is 378 2024-09-14T03:50:31.888 INFO:tasks.rados.rados.0.smithi110.stdout:3750: finishing copy_from to smithi11075562-10 2024-09-14T03:50:31.888 INFO:tasks.rados.rados.0.smithi110.stdout:3750: got expected ENOENT (src dne) 2024-09-14T03:50:31.888 INFO:tasks.rados.rados.0.smithi110.stdout:3753: seq_num 1159 ranges {0=212992} 2024-09-14T03:50:31.890 INFO:tasks.rados.rados.0.smithi110.stdout:3753: writing smithi11075562-5 from 0 to 212992 tid 1 2024-09-14T03:50:31.890 INFO:tasks.rados.rados.0.smithi110.stdout:3754: read oid 47 snap -1 2024-09-14T03:50:31.890 INFO:tasks.rados.rados.0.smithi110.stdout:3754: expect (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:50:31.890 INFO:tasks.rados.rados.0.smithi110.stdout:3755: rmattr oid 23 current snap is 378 2024-09-14T03:50:31.890 INFO:tasks.rados.rados.0.smithi110.stdout:3755: done (5 left) 2024-09-14T03:50:31.890 INFO:tasks.rados.rados.0.smithi110.stdout:3756: read oid 19 snap 371 2024-09-14T03:50:31.890 INFO:tasks.rados.rados.0.smithi110.stdout:3756: expect (ObjNum 1131 snap 367 seq_num 1131) 2024-09-14T03:50:31.890 INFO:tasks.rados.rados.0.smithi110.stdout:3757: read oid 42 snap -1 2024-09-14T03:50:31.891 INFO:tasks.rados.rados.0.smithi110.stdout:3757: expect (ObjNum 1144 snap 373 seq_num 1144) 2024-09-14T03:50:31.891 INFO:tasks.rados.rados.0.smithi110.stdout:3758: snap_create 2024-09-14T03:50:31.891 INFO:tasks.rados.rados.0.smithi110.stdout:3750: finishing copy_from racing read to smithi11075562-10 2024-09-14T03:50:31.891 INFO:tasks.rados.rados.0.smithi110.stdout:3751: finishing write tid 1 to smithi11075562-35 2024-09-14T03:50:31.891 INFO:tasks.rados.rados.0.smithi110.stdout:3751: finishing write tid 2 to smithi11075562-35 2024-09-14T03:50:31.891 INFO:tasks.rados.rados.0.smithi110.stdout:3751: finishing write tid 3 to smithi11075562-35 2024-09-14T03:50:31.891 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 2267 (ObjNum 1157 snap 378 seq_num 1157) dirty exists 2024-09-14T03:50:31.891 INFO:tasks.rados.rados.0.smithi110.stdout:3751: left oid 35 (ObjNum 1157 snap 378 seq_num 1157) 2024-09-14T03:50:31.897 INFO:tasks.rados.rados.0.smithi110.stdout:3752: finishing write tid 1 to smithi11075562-15 2024-09-14T03:50:31.897 INFO:tasks.rados.rados.0.smithi110.stdout:3752: finishing write tid 2 to smithi11075562-15 2024-09-14T03:50:31.898 INFO:tasks.rados.rados.0.smithi110.stdout:3752: finishing write tid 3 to smithi11075562-15 2024-09-14T03:50:31.898 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2521 (ObjNum 1158 snap 378 seq_num 1158) dirty exists 2024-09-14T03:50:31.898 INFO:tasks.rados.rados.0.smithi110.stdout:3752: left oid 15 (ObjNum 1158 snap 378 seq_num 1158) 2024-09-14T03:50:31.909 INFO:tasks.rados.rados.0.smithi110.stdout:3753: finishing write tid 1 to smithi11075562-5 2024-09-14T03:50:31.909 INFO:tasks.rados.rados.0.smithi110.stdout:3753: finishing write tid 2 to smithi11075562-5 2024-09-14T03:50:31.919 INFO:tasks.rados.rados.0.smithi110.stdout:3754: expect (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:50:31.961 INFO:tasks.rados.rados.0.smithi110.stdout:3753: finishing write tid 3 to smithi11075562-5 2024-09-14T03:50:31.961 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2583 (ObjNum 1159 snap 378 seq_num 1159) dirty exists 2024-09-14T03:50:31.961 INFO:tasks.rados.rados.0.smithi110.stdout:3753: left oid 5 (ObjNum 1159 snap 378 seq_num 1159) 2024-09-14T03:50:31.961 INFO:tasks.rados.rados.0.smithi110.stdout:3757: expect (ObjNum 1144 snap 373 seq_num 1144) 2024-09-14T03:50:32.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:31 smithi110 ceph-mon[88262]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T03:50:32.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:31 smithi110 ceph-mon[86423]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T03:50:32.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:31 smithi195 ceph-mon[82388]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T03:50:32.268 INFO:tasks.rados.rados.0.smithi110.stdout:3756: expect (ObjNum 1131 snap 367 seq_num 1131) 2024-09-14T03:50:32.876 INFO:tasks.rados.rados.0.smithi110.stdout:3750: done (7 left) 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:3751: done (6 left) 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:3752: done (5 left) 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:3753: done (4 left) 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:3754: done (3 left) 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:3756: done (2 left) 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:3757: done (1 left) 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:3758: done (0 left) 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 9 current snap is 379 2024-09-14T03:50:32.877 INFO:tasks.rados.rados.0.smithi110.stdout:3759: seq_num 1160 ranges {0=229376} 2024-09-14T03:50:32.878 INFO:tasks.rados.rados.0.smithi110.stdout:3759: writing smithi11075562-9 from 0 to 229376 tid 1 2024-09-14T03:50:32.878 INFO:tasks.rados.rados.0.smithi110.stdout:3760: setattr oid 3 current snap is 379 2024-09-14T03:50:32.878 INFO:tasks.rados.rados.0.smithi110.stdout:3761: rollback oid 19 current snap is 379 2024-09-14T03:50:32.878 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 19 to 366 2024-09-14T03:50:32.878 INFO:tasks.rados.rados.0.smithi110.stdout:3762: snap_remove snap 367 2024-09-14T03:50:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:32 smithi110 ceph-mon[88262]: pgmap v863: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 780 KiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-14T03:50:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:32 smithi110 ceph-mon[88262]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T03:50:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:32 smithi110 ceph-mon[86423]: pgmap v863: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 780 KiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-14T03:50:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:32 smithi110 ceph-mon[86423]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T03:50:33.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:32 smithi195 ceph-mon[82388]: pgmap v863: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 780 KiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-14T03:50:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:32 smithi195 ceph-mon[82388]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T03:50:33.833 INFO:tasks.rados.rados.0.smithi110.stdout:3762: done (3 left) 2024-09-14T03:50:33.833 INFO:tasks.rados.rados.0.smithi110.stdout:3763: rmattr oid 26 current snap is 379 2024-09-14T03:50:33.833 INFO:tasks.rados.rados.0.smithi110.stdout:3763: done (3 left) 2024-09-14T03:50:33.833 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 379 2024-09-14T03:50:33.833 INFO:tasks.rados.rados.0.smithi110.stdout:3764: seq_num 1161 ranges {1646592=245760} 2024-09-14T03:50:33.844 INFO:tasks.rados.rados.0.smithi110.stdout:3764: writing smithi11075562-15 from 1646592 to 1892352 tid 1 2024-09-14T03:50:33.844 INFO:tasks.rados.rados.0.smithi110.stdout:3765: rmattr oid 37 current snap is 379 2024-09-14T03:50:33.844 INFO:tasks.rados.rados.0.smithi110.stdout:3765: done (4 left) 2024-09-14T03:50:33.844 INFO:tasks.rados.rados.0.smithi110.stdout:3766: read oid 41 snap 359 2024-09-14T03:50:33.844 INFO:tasks.rados.rados.0.smithi110.stdout:3766: expect (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:50:33.844 INFO:tasks.rados.rados.0.smithi110.stdout:3767: copy_from oid 30 from oid 26 current snap is 379 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3761: finishing rollback tid 0 to smithi11075562-19 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 2152 (ObjNum 1124 snap 366 seq_num 1124) dirty exists 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3759: finishing write tid 1 to smithi11075562-9 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3759: finishing write tid 2 to smithi11075562-9 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3761: finishing rollback tid 1 to smithi11075562-19 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3761: finishing rollback tid 2 to smithi11075562-19 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2181 (ObjNum 1105 snap 350 seq_num 1105) dirty exists 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3759: finishing write tid 3 to smithi11075562-9 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 2149 (ObjNum 1160 snap 379 seq_num 1160) dirty exists 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3759: left oid 9 (ObjNum 1160 snap 379 seq_num 1160) 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3759: done (5 left) 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3760: done (4 left) 2024-09-14T03:50:33.845 INFO:tasks.rados.rados.0.smithi110.stdout:3761: done (3 left) 2024-09-14T03:50:33.846 INFO:tasks.rados.rados.0.smithi110.stdout:3768: read oid 25 snap -1 2024-09-14T03:50:33.846 INFO:tasks.rados.rados.0.smithi110.stdout:3768: expect (ObjNum 1154 snap 377 seq_num 1154) 2024-09-14T03:50:33.846 INFO:tasks.rados.rados.0.smithi110.stdout:3769: snap_remove snap 376 2024-09-14T03:50:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:33 smithi110 ceph-mon[88262]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T03:50:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:33 smithi110 ceph-mon[88262]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T03:50:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:33 smithi110 ceph-mon[86423]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T03:50:34.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:33 smithi110 ceph-mon[86423]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T03:50:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:33 smithi195 ceph-mon[82388]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T03:50:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:33 smithi195 ceph-mon[82388]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T03:50:34.836 INFO:tasks.rados.rados.0.smithi110.stdout:3764: finishing write tid 1 to smithi11075562-15 2024-09-14T03:50:34.836 INFO:tasks.rados.rados.0.smithi110.stdout:3764: finishing write tid 2 to smithi11075562-15 2024-09-14T03:50:34.837 INFO:tasks.rados.rados.0.smithi110.stdout:3764: finishing write tid 3 to smithi11075562-15 2024-09-14T03:50:34.837 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2523 (ObjNum 1161 snap 379 seq_num 1161) dirty exists 2024-09-14T03:50:34.837 INFO:tasks.rados.rados.0.smithi110.stdout:3764: left oid 15 (ObjNum 1161 snap 379 seq_num 1161) 2024-09-14T03:50:34.837 INFO:tasks.rados.rados.0.smithi110.stdout:3768: expect (ObjNum 1154 snap 377 seq_num 1154) 2024-09-14T03:50:35.039 INFO:tasks.rados.rados.0.smithi110.stdout:3766: expect (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:50:35.618 INFO:tasks.rados.rados.0.smithi110.stdout:3767: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:50:35.618 INFO:tasks.rados.rados.0.smithi110.stdout:3767: finishing copy_from to smithi11075562-30 2024-09-14T03:50:35.618 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 2015 (ObjNum 1145 snap 374 seq_num 1145) dirty exists 2024-09-14T03:50:35.618 INFO:tasks.rados.rados.0.smithi110.stdout:3764: done (4 left) 2024-09-14T03:50:35.618 INFO:tasks.rados.rados.0.smithi110.stdout:3766: done (3 left) 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3767: done (2 left) 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3768: done (1 left) 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3769: done (0 left) 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3770: rollback oid 33 current snap is 379 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 33 to 362 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3771: rollback oid 21 current snap is 379 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 365 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3772: copy_from oid 3 from oid 8 current snap is 379 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3773: setattr oid 26 current snap is 379 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3774: rollback oid 31 current snap is 379 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 31 to 354 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3775: read oid 34 snap -1 2024-09-14T03:50:35.619 INFO:tasks.rados.rados.0.smithi110.stdout:3775: expect deleted 2024-09-14T03:50:35.620 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 46 current snap is 379 2024-09-14T03:50:35.620 INFO:tasks.rados.rados.0.smithi110.stdout:3776: seq_num 1162 ranges {0=270336} 2024-09-14T03:50:35.620 INFO:tasks.rados.rados.0.smithi110.stdout:3776: writing smithi11075562-46 from 0 to 270336 tid 1 2024-09-14T03:50:35.620 INFO:tasks.rados.rados.0.smithi110.stdout:3777: read oid 50 snap -1 2024-09-14T03:50:35.620 INFO:tasks.rados.rados.0.smithi110.stdout:3777: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:50:35.620 INFO:tasks.rados.rados.0.smithi110.stdout:3778: snap_remove snap 359 2024-09-14T03:50:35.838 INFO:tasks.rados.rados.0.smithi110.stdout:3771: finishing rollback tid 0 to smithi11075562-21 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3774: finishing rollback tid 0 to smithi11075562-31 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3774: finishing rollback tid 1 to smithi11075562-31 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 2173 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 2016 (ObjNum 1145 snap 374 seq_num 1145) dirty exists 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3770: finishing rollback tid 0 to smithi11075562-33 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3770: finishing rollback tid 1 to smithi11075562-33 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 2151 (ObjNum 33 snap 0 seq_num 1533435502) dirty dne 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3776: finishing write tid 1 to smithi11075562-46 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3771: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3771: finishing rollback tid 2 to smithi11075562-21 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2184 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3776: finishing write tid 2 to smithi11075562-46 2024-09-14T03:50:35.839 INFO:tasks.rados.rados.0.smithi110.stdout:3776: finishing write tid 3 to smithi11075562-46 2024-09-14T03:50:35.840 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 2153 (ObjNum 1162 snap 379 seq_num 1162) dirty exists 2024-09-14T03:50:35.840 INFO:tasks.rados.rados.0.smithi110.stdout:3776: left oid 46 (ObjNum 1162 snap 379 seq_num 1162) 2024-09-14T03:50:35.840 INFO:tasks.rados.rados.0.smithi110.stdout:3777: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:50:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:35 smithi110 ceph-mon[86423]: pgmap v867: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 48 KiB/s wr, 3 op/s 2024-09-14T03:50:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:35 smithi110 ceph-mon[86423]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T03:50:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:35 smithi110 ceph-mon[88262]: pgmap v867: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 48 KiB/s wr, 3 op/s 2024-09-14T03:50:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:35 smithi110 ceph-mon[88262]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T03:50:36.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:35 smithi195 ceph-mon[82388]: pgmap v867: 121 pgs: 121 active+clean; 244 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 48 KiB/s wr, 3 op/s 2024-09-14T03:50:36.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:35 smithi195 ceph-mon[82388]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T03:50:36.290 INFO:tasks.rados.rados.0.smithi110.stdout:3772: finishing copy_from racing read to smithi11075562-3 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3772: finishing copy_from to smithi11075562-3 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 2519 (ObjNum 1076 snap 338 seq_num 1076) dirty exists 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3770: done (8 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3771: done (7 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3772: done (6 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3773: done (5 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3774: done (4 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3775: done (3 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3776: done (2 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3777: done (1 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3778: done (0 left) 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3779: copy_from oid 21 from oid 22 current snap is 379 2024-09-14T03:50:36.291 INFO:tasks.rados.rados.0.smithi110.stdout:3780: setattr oid 34 current snap is 379 2024-09-14T03:50:36.292 INFO:tasks.rados.rados.0.smithi110.stdout:3781: read oid 46 snap 372 2024-09-14T03:50:36.292 INFO:tasks.rados.rados.0.smithi110.stdout:3781: expect (ObjNum 66765376 snap 21934 seq_num 64985328) 2024-09-14T03:50:36.292 INFO:tasks.rados.rados.0.smithi110.stdout:3782: copy_from oid 30 from oid 37 current snap is 379 2024-09-14T03:50:36.292 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 17 current snap is 379 2024-09-14T03:50:36.293 INFO:tasks.rados.rados.0.smithi110.stdout:3783: seq_num 1163 ranges {0=286720} 2024-09-14T03:50:36.294 INFO:tasks.rados.rados.0.smithi110.stdout:3783: writing smithi11075562-17 from 0 to 286720 tid 1 2024-09-14T03:50:36.294 INFO:tasks.rados.rados.0.smithi110.stdout:3784: rmattr oid 9 current snap is 379 2024-09-14T03:50:36.294 INFO:tasks.rados.rados.0.smithi110.stdout:3784: done (5 left) 2024-09-14T03:50:36.294 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 28 current snap is 379 2024-09-14T03:50:36.295 INFO:tasks.rados.rados.0.smithi110.stdout:3781: expect (ObjNum 66765376 snap 21934 seq_num 64985328) 2024-09-14T03:50:36.299 INFO:tasks.rados.rados.0.smithi110.stdout:3782: finishing copy_from to smithi11075562-30 2024-09-14T03:50:36.299 INFO:tasks.rados.rados.0.smithi110.stdout:3782: got expected ENOENT (src dne) 2024-09-14T03:50:36.299 INFO:tasks.rados.rados.0.smithi110.stdout:3782: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:50:36.299 INFO:tasks.rados.rados.0.smithi110.stdout:3785: seq_num 1164 ranges {0=303104} 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:3785: writing smithi11075562-28 from 0 to 303104 tid 1 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:3781: done (5 left) 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:3782: done (4 left) 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:3786: snap_create 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 2174 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:3783: finishing write tid 1 to smithi11075562-17 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:3783: finishing write tid 2 to smithi11075562-17 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:3779: finishing copy_from racing read to smithi11075562-21 2024-09-14T03:50:36.301 INFO:tasks.rados.rados.0.smithi110.stdout:3779: finishing copy_from to smithi11075562-21 2024-09-14T03:50:36.302 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2502 (ObjNum 1102 snap 349 seq_num 1102) dirty exists 2024-09-14T03:50:36.302 INFO:tasks.rados.rados.0.smithi110.stdout:3783: finishing write tid 3 to smithi11075562-17 2024-09-14T03:50:36.302 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 17 v 2014 (ObjNum 1163 snap 379 seq_num 1163) dirty exists 2024-09-14T03:50:36.302 INFO:tasks.rados.rados.0.smithi110.stdout:3783: left oid 17 (ObjNum 1163 snap 379 seq_num 1163) 2024-09-14T03:50:36.303 INFO:tasks.rados.rados.0.smithi110.stdout:3785: finishing write tid 1 to smithi11075562-28 2024-09-14T03:50:36.303 INFO:tasks.rados.rados.0.smithi110.stdout:3785: finishing write tid 2 to smithi11075562-28 2024-09-14T03:50:36.304 INFO:tasks.rados.rados.0.smithi110.stdout:3785: finishing write tid 3 to smithi11075562-28 2024-09-14T03:50:36.304 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 2521 (ObjNum 1164 snap 379 seq_num 1164) dirty exists 2024-09-14T03:50:36.304 INFO:tasks.rados.rados.0.smithi110.stdout:3785: left oid 28 (ObjNum 1164 snap 379 seq_num 1164) 2024-09-14T03:50:36.850 INFO:tasks.rados.rados.0.smithi110.stdout:3779: done (4 left) 2024-09-14T03:50:36.851 INFO:tasks.rados.rados.0.smithi110.stdout:3780: done (3 left) 2024-09-14T03:50:36.851 INFO:tasks.rados.rados.0.smithi110.stdout:3783: done (2 left) 2024-09-14T03:50:36.851 INFO:tasks.rados.rados.0.smithi110.stdout:3785: done (1 left) 2024-09-14T03:50:36.851 INFO:tasks.rados.rados.0.smithi110.stdout:3786: done (0 left) 2024-09-14T03:50:36.851 INFO:tasks.rados.rados.0.smithi110.stdout:3787: snap_remove snap 362 2024-09-14T03:50:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:36 smithi110 ceph-mon[86423]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T03:50:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:36 smithi110 ceph-mon[86423]: pgmap v870: 121 pgs: 121 active+clean; 259 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 28 op/s 2024-09-14T03:50:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:36 smithi110 ceph-mon[88262]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T03:50:37.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:36 smithi110 ceph-mon[88262]: pgmap v870: 121 pgs: 121 active+clean; 259 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 28 op/s 2024-09-14T03:50:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:36 smithi195 ceph-mon[82388]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T03:50:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:36 smithi195 ceph-mon[82388]: pgmap v870: 121 pgs: 121 active+clean; 259 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 28 op/s 2024-09-14T03:50:37.854 INFO:tasks.rados.rados.0.smithi110.stdout:3787: done (0 left) 2024-09-14T03:50:37.855 INFO:tasks.rados.rados.0.smithi110.stdout:3788: copy_from oid 49 from oid 2 current snap is 380 2024-09-14T03:50:37.855 INFO:tasks.rados.rados.0.smithi110.stdout:3789: read oid 18 snap -1 2024-09-14T03:50:37.855 INFO:tasks.rados.rados.0.smithi110.stdout:3789: expect deleted 2024-09-14T03:50:37.855 INFO:tasks.rados.rados.0.smithi110.stdout:3790: copy_from oid 12 from oid 4 current snap is 380 2024-09-14T03:50:37.855 INFO:tasks.rados.rados.0.smithi110.stdout:3791: rollback oid 25 current snap is 380 2024-09-14T03:50:37.855 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 25 to 371 2024-09-14T03:50:37.855 INFO:tasks.rados.rados.0.smithi110.stdout:3792: snap_create 2024-09-14T03:50:37.875 INFO:tasks.rados.rados.0.smithi110.stdout:3791: finishing rollback tid 0 to smithi11075562-25 2024-09-14T03:50:37.875 INFO:tasks.rados.rados.0.smithi110.stdout:3791: finishing rollback tid 1 to smithi11075562-25 2024-09-14T03:50:37.875 INFO:tasks.rados.rados.0.smithi110.stdout:3791: finishing rollback tid 2 to smithi11075562-25 2024-09-14T03:50:37.875 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2609 (ObjNum 1109 snap 353 seq_num 1109) dirty exists 2024-09-14T03:50:37.878 INFO:tasks.rados.rados.0.smithi110.stdout:3788: finishing copy_from racing read to smithi11075562-49 2024-09-14T03:50:37.879 INFO:tasks.rados.rados.0.smithi110.stdout:3788: finishing copy_from to smithi11075562-49 2024-09-14T03:50:37.879 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2151 (ObjNum 1066 snap 336 seq_num 1066) dirty exists 2024-09-14T03:50:37.920 INFO:tasks.rados.rados.0.smithi110.stdout:3790: finishing copy_from racing read to smithi11075562-12 2024-09-14T03:50:37.922 INFO:tasks.rados.rados.0.smithi110.stdout:3790: finishing copy_from to smithi11075562-12 2024-09-14T03:50:37.922 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2166 (ObjNum 1113 snap 354 seq_num 1113) dirty exists 2024-09-14T03:50:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:37 smithi110 ceph-mon[88262]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T03:50:38.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:37 smithi110 ceph-mon[86423]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T03:50:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:37 smithi195 ceph-mon[82388]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T03:50:38.834 INFO:tasks.rados.rados.0.smithi110.stdout:3788: done (4 left) 2024-09-14T03:50:38.834 INFO:tasks.rados.rados.0.smithi110.stdout:3789: done (3 left) 2024-09-14T03:50:38.834 INFO:tasks.rados.rados.0.smithi110.stdout:3790: done (2 left) 2024-09-14T03:50:38.835 INFO:tasks.rados.rados.0.smithi110.stdout:3791: done (1 left) 2024-09-14T03:50:38.835 INFO:tasks.rados.rados.0.smithi110.stdout:3792: done (0 left) 2024-09-14T03:50:38.835 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 381 2024-09-14T03:50:38.835 INFO:tasks.rados.rados.0.smithi110.stdout:3793: seq_num 1165 ranges {0=319488} 2024-09-14T03:50:38.835 INFO:tasks.rados.rados.0.smithi110.stdout:3793: writing smithi11075562-32 from 0 to 319488 tid 1 2024-09-14T03:50:38.835 INFO:tasks.rados.rados.0.smithi110.stdout:3794: snap_remove snap 352 2024-09-14T03:50:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:38 smithi110 ceph-mon[88262]: pgmap v873: 121 pgs: 121 active+clean; 259 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 4.0 MiB/s wr, 30 op/s 2024-09-14T03:50:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:38 smithi110 ceph-mon[88262]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T03:50:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:38 smithi110 ceph-mon[88262]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T03:50:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:38 smithi110 ceph-mon[86423]: pgmap v873: 121 pgs: 121 active+clean; 259 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 4.0 MiB/s wr, 30 op/s 2024-09-14T03:50:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:38 smithi110 ceph-mon[86423]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T03:50:39.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:38 smithi110 ceph-mon[86423]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T03:50:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:38 smithi195 ceph-mon[82388]: pgmap v873: 121 pgs: 121 active+clean; 259 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 4.0 MiB/s wr, 30 op/s 2024-09-14T03:50:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:38 smithi195 ceph-mon[82388]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T03:50:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:38 smithi195 ceph-mon[82388]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T03:50:39.838 INFO:tasks.rados.rados.0.smithi110.stdout:3794: done (1 left) 2024-09-14T03:50:39.838 INFO:tasks.rados.rados.0.smithi110.stdout:3795: copy_from oid 44 from oid 3 current snap is 381 2024-09-14T03:50:39.838 INFO:tasks.rados.rados.0.smithi110.stdout:3796: read oid 39 snap -1 2024-09-14T03:50:39.838 INFO:tasks.rados.rados.0.smithi110.stdout:3796: expect deleted 2024-09-14T03:50:39.838 INFO:tasks.rados.rados.0.smithi110.stdout:3797: copy_from oid 36 from oid 14 current snap is 381 2024-09-14T03:50:39.838 INFO:tasks.rados.rados.0.smithi110.stdout:3798: rmattr oid 25 current snap is 381 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3798: done (4 left) 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3799: read oid 41 snap -1 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3799: expect (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3800: read oid 38 snap -1 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3800: expect (ObjNum 1127 snap 366 seq_num 1127) 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3801: copy_from oid 25 from oid 9 current snap is 381 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3802: setattr oid 16 current snap is 381 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3793: finishing write tid 1 to smithi11075562-32 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3793: finishing write tid 2 to smithi11075562-32 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3793: finishing write tid 3 to smithi11075562-32 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2525 (ObjNum 1165 snap 381 seq_num 1165) dirty exists 2024-09-14T03:50:39.839 INFO:tasks.rados.rados.0.smithi110.stdout:3793: left oid 32 (ObjNum 1165 snap 381 seq_num 1165) 2024-09-14T03:50:39.840 INFO:tasks.rados.rados.0.smithi110.stdout:3793: done (7 left) 2024-09-14T03:50:39.840 INFO:tasks.rados.rados.0.smithi110.stdout:3803: delete oid 27 current snap is 381 2024-09-14T03:50:39.852 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2182 (ObjNum 65 snap 0 seq_num 1733647724) dirty exists 2024-09-14T03:50:39.860 INFO:tasks.rados.rados.0.smithi110.stdout:3796: done (7 left) 2024-09-14T03:50:39.860 INFO:tasks.rados.rados.0.smithi110.stdout:3802: done (6 left) 2024-09-14T03:50:39.860 INFO:tasks.rados.rados.0.smithi110.stdout:3803: done (5 left) 2024-09-14T03:50:39.860 INFO:tasks.rados.rados.0.smithi110.stdout:3804: copy_from oid 16 from oid 1 current snap is 381 2024-09-14T03:50:39.861 INFO:tasks.rados.rados.0.smithi110.stdout:3805: copy_from oid 7 from oid 31 current snap is 381 2024-09-14T03:50:39.861 INFO:tasks.rados.rados.0.smithi110.stdout:3806: delete oid 40 current snap is 381 2024-09-14T03:50:39.864 INFO:tasks.rados.rados.0.smithi110.stdout:3800: expect (ObjNum 1127 snap 366 seq_num 1127) 2024-09-14T03:50:40.053 INFO:tasks.rados.rados.0.smithi110.stdout:3801: finishing copy_from racing read to smithi11075562-25 2024-09-14T03:50:40.053 INFO:tasks.rados.rados.0.smithi110.stdout:3801: finishing copy_from to smithi11075562-25 2024-09-14T03:50:40.053 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2149 (ObjNum 1160 snap 379 seq_num 1160) dirty exists 2024-09-14T03:50:40.053 INFO:tasks.rados.rados.0.smithi110.stdout:3800: done (7 left) 2024-09-14T03:50:40.053 INFO:tasks.rados.rados.0.smithi110.stdout:3801: done (6 left) 2024-09-14T03:50:40.054 INFO:tasks.rados.rados.0.smithi110.stdout:3806: done (5 left) 2024-09-14T03:50:40.054 INFO:tasks.rados.rados.0.smithi110.stdout:3807: snap_remove snap 379 2024-09-14T03:50:40.196 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:50:39 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[93555]: 2024-09-14T03:50:39.862+0000 7f099b023640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1af after trimming was started 2024-09-14T03:50:40.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:50:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:50:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:40.844 INFO:tasks.rados.rados.0.smithi110.stdout:3807: done (5 left) 2024-09-14T03:50:40.844 INFO:tasks.rados.rados.0.smithi110.stdout:3808: setattr oid 6 current snap is 381 2024-09-14T03:50:40.844 INFO:tasks.rados.rados.0.smithi110.stdout:3805: finishing copy_from to smithi11075562-7 2024-09-14T03:50:40.844 INFO:tasks.rados.rados.0.smithi110.stdout:3805: got expected ENOENT (src dne) 2024-09-14T03:50:40.844 INFO:tasks.rados.rados.0.smithi110.stdout:3805: finishing copy_from racing read to smithi11075562-7 2024-09-14T03:50:40.844 INFO:tasks.rados.rados.0.smithi110.stdout:3795: finishing copy_from racing read to smithi11075562-44 2024-09-14T03:50:40.844 INFO:tasks.rados.rados.0.smithi110.stdout:3795: finishing copy_from to smithi11075562-44 2024-09-14T03:50:40.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 2519 (ObjNum 1076 snap 338 seq_num 1076) dirty exists 2024-09-14T03:50:40.845 INFO:tasks.rados.rados.0.smithi110.stdout:3804: finishing copy_from racing read to smithi11075562-16 2024-09-14T03:50:40.845 INFO:tasks.rados.rados.0.smithi110.stdout:3797: finishing copy_from racing read to smithi11075562-36 2024-09-14T03:50:40.845 INFO:tasks.rados.rados.0.smithi110.stdout:3804: finishing copy_from to smithi11075562-16 2024-09-14T03:50:40.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2264 (ObjNum 1153 snap 377 seq_num 1153) dirty exists 2024-09-14T03:50:40.845 INFO:tasks.rados.rados.0.smithi110.stdout:3797: finishing copy_from to smithi11075562-36 2024-09-14T03:50:40.845 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2175 (ObjNum 1137 snap 368 seq_num 1137) dirty exists 2024-09-14T03:50:40.845 INFO:tasks.rados.rados.0.smithi110.stdout:3799: expect (ObjNum 1008 snap 323 seq_num 1008) 2024-09-14T03:50:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:40 smithi110 ceph-mon[88262]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T03:50:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:40 smithi110 ceph-mon[86423]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T03:50:41.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:40 smithi195 ceph-mon[82388]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T03:50:41.428 INFO:tasks.rados.rados.0.smithi110.stdout:3795: done (5 left) 2024-09-14T03:50:41.429 INFO:tasks.rados.rados.0.smithi110.stdout:3797: done (4 left) 2024-09-14T03:50:41.429 INFO:tasks.rados.rados.0.smithi110.stdout:3799: done (3 left) 2024-09-14T03:50:41.429 INFO:tasks.rados.rados.0.smithi110.stdout:3804: done (2 left) 2024-09-14T03:50:41.429 INFO:tasks.rados.rados.0.smithi110.stdout:3805: done (1 left) 2024-09-14T03:50:41.430 INFO:tasks.rados.rados.0.smithi110.stdout:3809: delete oid 33 current snap is 381 2024-09-14T03:50:41.431 INFO:tasks.rados.rados.0.smithi110.stdout:3809: done (1 left) 2024-09-14T03:50:41.432 INFO:tasks.rados.rados.0.smithi110.stdout:3810: read oid 48 snap -1 2024-09-14T03:50:41.432 INFO:tasks.rados.rados.0.smithi110.stdout:3810: expect deleted 2024-09-14T03:50:41.432 INFO:tasks.rados.rados.0.smithi110.stdout:3811: rollback oid 21 current snap is 381 2024-09-14T03:50:41.432 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 21 to 369 2024-09-14T03:50:41.432 INFO:tasks.rados.rados.0.smithi110.stdout:3812: read oid 39 snap -1 2024-09-14T03:50:41.432 INFO:tasks.rados.rados.0.smithi110.stdout:3812: expect deleted 2024-09-14T03:50:41.432 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 32 current snap is 381 2024-09-14T03:50:41.433 INFO:tasks.rados.rados.0.smithi110.stdout:3813: seq_num 1169 ranges {319488=385024} 2024-09-14T03:50:41.437 INFO:tasks.rados.rados.0.smithi110.stdout:3813: writing smithi11075562-32 from 319488 to 704512 tid 1 2024-09-14T03:50:41.437 INFO:tasks.rados.rados.0.smithi110.stdout:3814: rollback oid 11 current snap is 381 2024-09-14T03:50:41.437 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 361 2024-09-14T03:50:41.437 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2265 (ObjNum 953 snap 307 seq_num 953) dirty exists 2024-09-14T03:50:41.437 INFO:tasks.rados.rados.0.smithi110.stdout:3808: done (5 left) 2024-09-14T03:50:41.437 INFO:tasks.rados.rados.0.smithi110.stdout:3810: done (4 left) 2024-09-14T03:50:41.437 INFO:tasks.rados.rados.0.smithi110.stdout:3815: copy_from oid 7 from oid 33 current snap is 381 2024-09-14T03:50:41.438 INFO:tasks.rados.rados.0.smithi110.stdout:3816: read oid 29 snap -1 2024-09-14T03:50:41.438 INFO:tasks.rados.rados.0.smithi110.stdout:3816: expect (ObjNum 1112 snap 353 seq_num 1112) 2024-09-14T03:50:41.438 INFO:tasks.rados.rados.0.smithi110.stdout:3817: copy_from oid 19 from oid 28 current snap is 381 2024-09-14T03:50:41.438 INFO:tasks.rados.rados.0.smithi110.stdout:3818: snap_remove snap 378 2024-09-14T03:50:41.838 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:50:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:50:41] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-14T03:50:41.846 INFO:tasks.rados.rados.0.smithi110.stdout:3818: done (7 left) 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3819: rollback oid 6 current snap is 381 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 6 to 380 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3820: copy_from oid 23 from oid 12 current snap is 381 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3811: finishing rollback tid 0 to smithi11075562-21 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3811: finishing rollback tid 1 to smithi11075562-21 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3811: finishing rollback tid 2 to smithi11075562-21 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2505 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3815: finishing copy_from to smithi11075562-7 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3815: got expected ENOENT (src dne) 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3815: finishing copy_from racing read to smithi11075562-7 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3814: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:50:41.847 INFO:tasks.rados.rados.0.smithi110.stdout:3814: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:3813: finishing write tid 1 to smithi11075562-32 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:3813: finishing write tid 2 to smithi11075562-32 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:3814: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 2017 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:3813: finishing write tid 3 to smithi11075562-32 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 32 v 2527 (ObjNum 1169 snap 381 seq_num 1169) dirty exists 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:3813: left oid 32 (ObjNum 1169 snap 381 seq_num 1169) 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:3817: finishing copy_from racing read to smithi11075562-19 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:3817: finishing copy_from to smithi11075562-19 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2521 (ObjNum 1164 snap 379 seq_num 1164) dirty exists 2024-09-14T03:50:41.848 INFO:tasks.rados.rados.0.smithi110.stdout:3816: expect (ObjNum 1112 snap 353 seq_num 1112) 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3819: finishing rollback tid 0 to smithi11075562-6 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3819: finishing rollback tid 1 to smithi11075562-6 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3819: finishing rollback tid 2 to smithi11075562-6 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2524 (ObjNum 953 snap 307 seq_num 953) dirty exists 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3820: finishing copy_from racing read to smithi11075562-23 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3820: finishing copy_from to smithi11075562-23 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 23 v 2166 (ObjNum 1113 snap 354 seq_num 1113) dirty exists 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3811: done (8 left) 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3812: done (7 left) 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3813: done (6 left) 2024-09-14T03:50:41.975 INFO:tasks.rados.rados.0.smithi110.stdout:3814: done (5 left) 2024-09-14T03:50:41.976 INFO:tasks.rados.rados.0.smithi110.stdout:3815: done (4 left) 2024-09-14T03:50:41.976 INFO:tasks.rados.rados.0.smithi110.stdout:3816: done (3 left) 2024-09-14T03:50:41.976 INFO:tasks.rados.rados.0.smithi110.stdout:3817: done (2 left) 2024-09-14T03:50:41.976 INFO:tasks.rados.rados.0.smithi110.stdout:3819: done (1 left) 2024-09-14T03:50:41.976 INFO:tasks.rados.rados.0.smithi110.stdout:3820: done (0 left) 2024-09-14T03:50:41.976 INFO:tasks.rados.rados.0.smithi110.stdout:3821: snap_remove snap 354 2024-09-14T03:50:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:41 smithi110 ceph-mon[86423]: pgmap v876: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 266 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-09-14T03:50:42.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:41 smithi110 ceph-mon[86423]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T03:50:42.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:41 smithi110 ceph-mon[88262]: pgmap v876: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 266 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-09-14T03:50:42.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:41 smithi110 ceph-mon[88262]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T03:50:42.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:41 smithi195 ceph-mon[82388]: pgmap v876: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 266 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-09-14T03:50:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:41 smithi195 ceph-mon[82388]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T03:50:42.851 INFO:tasks.rados.rados.0.smithi110.stdout:3821: done (0 left) 2024-09-14T03:50:42.851 INFO:tasks.rados.rados.0.smithi110.stdout:3822: rmattr oid 21 current snap is 381 2024-09-14T03:50:42.852 INFO:tasks.rados.rados.0.smithi110.stdout:3823: read oid 26 snap -1 2024-09-14T03:50:42.852 INFO:tasks.rados.rados.0.smithi110.stdout:3823: expect (ObjNum 1145 snap 374 seq_num 1145) 2024-09-14T03:50:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 10 current snap is 381 2024-09-14T03:50:42.853 INFO:tasks.rados.rados.0.smithi110.stdout:3824: seq_num 1170 ranges {6643712=401408} 2024-09-14T03:50:42.892 INFO:tasks.rados.rados.0.smithi110.stdout:3824: writing smithi11075562-10 from 6643712 to 7045120 tid 1 2024-09-14T03:50:42.892 INFO:tasks.rados.rados.0.smithi110.stdout:3825: snap_create 2024-09-14T03:50:42.892 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2506 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-14T03:50:42.896 INFO:tasks.rados.rados.0.smithi110.stdout:3824: finishing write tid 1 to smithi11075562-10 2024-09-14T03:50:42.896 INFO:tasks.rados.rados.0.smithi110.stdout:3824: finishing write tid 2 to smithi11075562-10 2024-09-14T03:50:42.898 INFO:tasks.rados.rados.0.smithi110.stdout:3824: finishing write tid 3 to smithi11075562-10 2024-09-14T03:50:42.898 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 10 v 2168 (ObjNum 1170 snap 381 seq_num 1170) dirty exists 2024-09-14T03:50:42.898 INFO:tasks.rados.rados.0.smithi110.stdout:3824: left oid 10 (ObjNum 1170 snap 381 seq_num 1170) 2024-09-14T03:50:42.904 INFO:tasks.rados.rados.0.smithi110.stdout:3823: expect (ObjNum 1145 snap 374 seq_num 1145) 2024-09-14T03:50:43.110 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:50:42 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:50:42.888+0000 7f0e0b357640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1b1 after trimming was started 2024-09-14T03:50:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:42 smithi195 ceph-mon[82388]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T03:50:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:42 smithi195 ceph-mon[82388]: pgmap v879: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 289 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.8 MiB/s rd, 7.6 MiB/s wr, 26 op/s 2024-09-14T03:50:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:42 smithi110 ceph-mon[86423]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T03:50:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:42 smithi110 ceph-mon[86423]: pgmap v879: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 289 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.8 MiB/s rd, 7.6 MiB/s wr, 26 op/s 2024-09-14T03:50:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:42 smithi110 ceph-mon[88262]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T03:50:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:42 smithi110 ceph-mon[88262]: pgmap v879: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 289 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.8 MiB/s rd, 7.6 MiB/s wr, 26 op/s 2024-09-14T03:50:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:43.838 INFO:tasks.rados.rados.0.smithi110.stdout:3822: done (3 left) 2024-09-14T03:50:43.838 INFO:tasks.rados.rados.0.smithi110.stdout:3823: done (2 left) 2024-09-14T03:50:43.838 INFO:tasks.rados.rados.0.smithi110.stdout:3824: done (1 left) 2024-09-14T03:50:43.838 INFO:tasks.rados.rados.0.smithi110.stdout:3825: done (0 left) 2024-09-14T03:50:43.838 INFO:tasks.rados.rados.0.smithi110.stdout:3826: delete oid 39 current snap is 382 2024-09-14T03:50:43.839 INFO:tasks.rados.rados.0.smithi110.stdout:3826: done (0 left) 2024-09-14T03:50:43.839 INFO:tasks.rados.rados.0.smithi110.stdout:3827: read oid 44 snap 370 2024-09-14T03:50:43.839 INFO:tasks.rados.rados.0.smithi110.stdout:3827: expect (ObjNum 1088 snap 343 seq_num 1088) 2024-09-14T03:50:43.839 INFO:tasks.rados.rados.0.smithi110.stdout:3828: copy_from oid 18 from oid 11 current snap is 382 2024-09-14T03:50:43.839 INFO:tasks.rados.rados.0.smithi110.stdout:3829: read oid 20 snap -1 2024-09-14T03:50:43.839 INFO:tasks.rados.rados.0.smithi110.stdout:3829: expect deleted 2024-09-14T03:50:43.839 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 4 current snap is 382 2024-09-14T03:50:43.839 INFO:tasks.rados.rados.0.smithi110.stdout:3830: seq_num 1172 ranges {12967936=434176} 2024-09-14T03:50:43.911 INFO:tasks.rados.rados.0.smithi110.stdout:3830: writing smithi11075562-4 from 12967936 to 13402112 tid 1 2024-09-14T03:50:43.911 INFO:tasks.rados.rados.0.smithi110.stdout:3831: snap_create 2024-09-14T03:50:43.911 INFO:tasks.rados.rados.0.smithi110.stdout:3828: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:50:43.911 INFO:tasks.rados.rados.0.smithi110.stdout:3828: finishing copy_from to smithi11075562-18 2024-09-14T03:50:43.911 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 2017 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:50:43.912 INFO:tasks.rados.rados.0.smithi110.stdout:3827: expect (ObjNum 1088 snap 343 seq_num 1088) 2024-09-14T03:50:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:43 smithi110 ceph-mon[86423]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T03:50:44.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:43 smithi110 ceph-mon[86423]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T03:50:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:43 smithi110 ceph-mon[88262]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T03:50:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:43 smithi110 ceph-mon[88262]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T03:50:44.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:43 smithi195 ceph-mon[82388]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T03:50:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:43 smithi195 ceph-mon[82388]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T03:50:44.653 INFO:tasks.rados.rados.0.smithi110.stdout:3830: finishing write tid 1 to smithi11075562-4 2024-09-14T03:50:44.653 INFO:tasks.rados.rados.0.smithi110.stdout:3830: finishing write tid 2 to smithi11075562-4 2024-09-14T03:50:44.654 INFO:tasks.rados.rados.0.smithi110.stdout:3830: finishing write tid 3 to smithi11075562-4 2024-09-14T03:50:44.654 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2176 (ObjNum 1172 snap 382 seq_num 1172) dirty exists 2024-09-14T03:50:44.654 INFO:tasks.rados.rados.0.smithi110.stdout:3830: left oid 4 (ObjNum 1172 snap 382 seq_num 1172) 2024-09-14T03:50:44.856 INFO:tasks.rados.rados.0.smithi110.stdout:3827: done (4 left) 2024-09-14T03:50:44.857 INFO:tasks.rados.rados.0.smithi110.stdout:3828: done (3 left) 2024-09-14T03:50:44.857 INFO:tasks.rados.rados.0.smithi110.stdout:3829: done (2 left) 2024-09-14T03:50:44.857 INFO:tasks.rados.rados.0.smithi110.stdout:3830: done (1 left) 2024-09-14T03:50:44.857 INFO:tasks.rados.rados.0.smithi110.stdout:3831: done (0 left) 2024-09-14T03:50:44.857 INFO:tasks.rados.rados.0.smithi110.stdout:3832: setattr oid 16 current snap is 383 2024-09-14T03:50:44.858 INFO:tasks.rados.rados.0.smithi110.stdout:3833: read oid 45 snap -1 2024-09-14T03:50:44.858 INFO:tasks.rados.rados.0.smithi110.stdout:3833: expect deleted 2024-09-14T03:50:44.858 INFO:tasks.rados.rados.0.smithi110.stdout:3834: snap_create 2024-09-14T03:50:44.862 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2525 (ObjNum 1153 snap 377 seq_num 1153) dirty exists 2024-09-14T03:50:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:44 smithi110 ceph-mon[86423]: pgmap v882: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 300 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 17 op/s 2024-09-14T03:50:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:44 smithi110 ceph-mon[88262]: pgmap v882: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 300 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 17 op/s 2024-09-14T03:50:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:44 smithi195 ceph-mon[82388]: pgmap v882: 121 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 117 active+clean; 300 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 17 op/s 2024-09-14T03:50:45.859 INFO:tasks.rados.rados.0.smithi110.stdout:3832: done (2 left) 2024-09-14T03:50:45.859 INFO:tasks.rados.rados.0.smithi110.stdout:3833: done (1 left) 2024-09-14T03:50:45.859 INFO:tasks.rados.rados.0.smithi110.stdout:3834: done (0 left) 2024-09-14T03:50:45.860 INFO:tasks.rados.rados.0.smithi110.stdout:3835: setattr oid 41 current snap is 384 2024-09-14T03:50:45.861 INFO:tasks.rados.rados.0.smithi110.stdout:3836: rollback oid 2 current snap is 384 2024-09-14T03:50:45.861 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 2 to 382 2024-09-14T03:50:45.861 INFO:tasks.rados.rados.0.smithi110.stdout:3837: snap_remove snap 349 2024-09-14T03:50:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:45 smithi110 ceph-mon[86423]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T03:50:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:45 smithi110 ceph-mon[88262]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T03:50:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:45 smithi195 ceph-mon[82388]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T03:50:46.873 INFO:tasks.rados.rados.0.smithi110.stdout:3837: done (2 left) 2024-09-14T03:50:46.873 INFO:tasks.rados.rados.0.smithi110.stdout:3838: rmattr oid 1 current snap is 384 2024-09-14T03:50:46.873 INFO:tasks.rados.rados.0.smithi110.stdout:3838: done (2 left) 2024-09-14T03:50:46.873 INFO:tasks.rados.rados.0.smithi110.stdout:3839: snap_remove snap 368 2024-09-14T03:50:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:46 smithi110 ceph-mon[88262]: pgmap v884: 121 pgs: 121 active+clean; 314 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 8 op/s 2024-09-14T03:50:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:46 smithi110 ceph-mon[88262]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T03:50:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:46 smithi110 ceph-mon[86423]: pgmap v884: 121 pgs: 121 active+clean; 314 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 8 op/s 2024-09-14T03:50:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:46 smithi110 ceph-mon[86423]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T03:50:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:46 smithi195 ceph-mon[82388]: pgmap v884: 121 pgs: 121 active+clean; 314 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 8 op/s 2024-09-14T03:50:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:46 smithi195 ceph-mon[82388]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T03:50:47.881 INFO:tasks.rados.rados.0.smithi110.stdout:3839: done (2 left) 2024-09-14T03:50:47.881 INFO:tasks.rados.rados.0.smithi110.stdout:3840: snap_create 2024-09-14T03:50:47.881 INFO:tasks.rados.rados.0.smithi110.stdout:3836: finishing rollback tid 0 to smithi11075562-2 2024-09-14T03:50:47.881 INFO:tasks.rados.rados.0.smithi110.stdout:3836: finishing rollback tid 1 to smithi11075562-2 2024-09-14T03:50:47.881 INFO:tasks.rados.rados.0.smithi110.stdout:3836: finishing rollback tid 2 to smithi11075562-2 2024-09-14T03:50:47.881 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 2524 (ObjNum 1066 snap 336 seq_num 1066) dirty exists 2024-09-14T03:50:47.881 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 2507 (ObjNum 1008 snap 323 seq_num 1008) dirty exists 2024-09-14T03:50:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:47 smithi110 ceph-mon[88262]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T03:50:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:47 smithi110 ceph-mon[86423]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T03:50:48.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:47 smithi195 ceph-mon[82388]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T03:50:48.837 INFO:tasks.rados.rados.0.smithi110.stdout:3835: done (2 left) 2024-09-14T03:50:48.837 INFO:tasks.rados.rados.0.smithi110.stdout:3836: done (1 left) 2024-09-14T03:50:48.837 INFO:tasks.rados.rados.0.smithi110.stdout:3840: done (0 left) 2024-09-14T03:50:48.837 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 385 2024-09-14T03:50:48.837 INFO:tasks.rados.rados.0.smithi110.stdout:3841: seq_num 1173 ranges {3481600=450560} 2024-09-14T03:50:48.857 INFO:tasks.rados.rados.0.smithi110.stdout:3841: writing smithi11075562-29 from 3481600 to 3932160 tid 1 2024-09-14T03:50:48.858 INFO:tasks.rados.rados.0.smithi110.stdout:3842: delete oid 37 current snap is 385 2024-09-14T03:50:48.863 INFO:tasks.rados.rados.0.smithi110.stdout:3842: done (1 left) 2024-09-14T03:50:48.863 INFO:tasks.rados.rados.0.smithi110.stdout:3843: copy_from oid 33 from oid 15 current snap is 385 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3844: read oid 20 snap 372 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3844: expect deleted 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3845: copy_from oid 39 from oid 7 current snap is 385 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3846: snap_create 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3845: finishing copy_from to smithi11075562-39 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3845: got expected ENOENT (src dne) 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3845: finishing copy_from racing read to smithi11075562-39 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3841: finishing write tid 1 to smithi11075562-29 2024-09-14T03:50:48.864 INFO:tasks.rados.rados.0.smithi110.stdout:3841: finishing write tid 2 to smithi11075562-29 2024-09-14T03:50:48.865 INFO:tasks.rados.rados.0.smithi110.stdout:3841: finishing write tid 3 to smithi11075562-29 2024-09-14T03:50:48.865 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1656 (ObjNum 1173 snap 385 seq_num 1173) dirty exists 2024-09-14T03:50:48.865 INFO:tasks.rados.rados.0.smithi110.stdout:3841: left oid 29 (ObjNum 1173 snap 385 seq_num 1173) 2024-09-14T03:50:48.875 INFO:tasks.rados.rados.0.smithi110.stdout:3843: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:50:48.876 INFO:tasks.rados.rados.0.smithi110.stdout:3843: finishing copy_from to smithi11075562-33 2024-09-14T03:50:48.876 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 2523 (ObjNum 1161 snap 379 seq_num 1161) dirty exists 2024-09-14T03:50:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:48 smithi110 ceph-mon[86423]: pgmap v887: 121 pgs: 121 active+clean; 309 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-14T03:50:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:48 smithi110 ceph-mon[86423]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T03:50:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:48 smithi110 ceph-mon[86423]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T03:50:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:48 smithi110 ceph-mon[88262]: pgmap v887: 121 pgs: 121 active+clean; 309 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-14T03:50:49.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:48 smithi110 ceph-mon[88262]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T03:50:49.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:48 smithi110 ceph-mon[88262]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T03:50:49.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:48 smithi195 ceph-mon[82388]: pgmap v887: 121 pgs: 121 active+clean; 309 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-14T03:50:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:48 smithi195 ceph-mon[82388]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T03:50:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:48 smithi195 ceph-mon[82388]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T03:50:49.838 INFO:tasks.rados.rados.0.smithi110.stdout:3841: done (4 left) 2024-09-14T03:50:49.838 INFO:tasks.rados.rados.0.smithi110.stdout:3843: done (3 left) 2024-09-14T03:50:49.838 INFO:tasks.rados.rados.0.smithi110.stdout:3844: done (2 left) 2024-09-14T03:50:49.838 INFO:tasks.rados.rados.0.smithi110.stdout:3845: done (1 left) 2024-09-14T03:50:49.838 INFO:tasks.rados.rados.0.smithi110.stdout:3846: done (0 left) 2024-09-14T03:50:49.838 INFO:tasks.rados.rados.0.smithi110.stdout:3847: delete oid 8 current snap is 386 2024-09-14T03:50:49.845 INFO:tasks.rados.rados.0.smithi110.stdout:3847: done (0 left) 2024-09-14T03:50:49.845 INFO:tasks.rados.rados.0.smithi110.stdout:3848: snap_remove snap 361 2024-09-14T03:50:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:50:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:50:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:50.847 INFO:tasks.rados.rados.0.smithi110.stdout:3848: done (0 left) 2024-09-14T03:50:50.848 INFO:tasks.rados.rados.0.smithi110.stdout:3849: rmattr oid 5 current snap is 386 2024-09-14T03:50:50.848 INFO:tasks.rados.rados.0.smithi110.stdout:3849: done (0 left) 2024-09-14T03:50:50.848 INFO:tasks.rados.rados.0.smithi110.stdout:3850: snap_create 2024-09-14T03:50:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:50 smithi110 ceph-mon[88262]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T03:50:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:50 smithi110 ceph-mon[86423]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T03:50:51.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:50 smithi195 ceph-mon[82388]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T03:50:51.840 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:50:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:50:51] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.51.0" 2024-09-14T03:50:51.848 INFO:tasks.rados.rados.0.smithi110.stdout:3850: done (0 left) 2024-09-14T03:50:51.848 INFO:tasks.rados.rados.0.smithi110.stdout:3851: snap_remove snap 372 2024-09-14T03:50:52.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:51 smithi110 ceph-mon[88262]: pgmap v891: 121 pgs: 121 active+clean; 299 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 596 KiB/s wr, 12 op/s 2024-09-14T03:50:52.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:51 smithi110 ceph-mon[88262]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T03:50:52.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:51 smithi110 ceph-mon[86423]: pgmap v891: 121 pgs: 121 active+clean; 299 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 596 KiB/s wr, 12 op/s 2024-09-14T03:50:52.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:51 smithi110 ceph-mon[86423]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T03:50:52.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:51 smithi195 ceph-mon[82388]: pgmap v891: 121 pgs: 121 active+clean; 299 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 596 KiB/s wr, 12 op/s 2024-09-14T03:50:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:51 smithi195 ceph-mon[82388]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T03:50:52.854 INFO:tasks.rados.rados.0.smithi110.stdout:3851: done (0 left) 2024-09-14T03:50:52.854 INFO:tasks.rados.rados.0.smithi110.stdout:3852: snap_remove snap 366 2024-09-14T03:50:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:52 smithi110 ceph-mon[88262]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T03:50:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:52 smithi110 ceph-mon[88262]: pgmap v894: 121 pgs: 121 active+clean; 287 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:50:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:52 smithi110 ceph-mon[86423]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T03:50:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:52 smithi110 ceph-mon[86423]: pgmap v894: 121 pgs: 121 active+clean; 287 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:50:53.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:52 smithi195 ceph-mon[82388]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T03:50:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:52 smithi195 ceph-mon[82388]: pgmap v894: 121 pgs: 121 active+clean; 287 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:50:53.839 INFO:tasks.rados.rados.0.smithi110.stdout:3852: done (0 left) 2024-09-14T03:50:53.839 INFO:tasks.rados.rados.0.smithi110.stdout:3853: snap_create 2024-09-14T03:50:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:53 smithi110 ceph-mon[88262]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T03:50:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:53 smithi110 ceph-mon[88262]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T03:50:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:53 smithi110 ceph-mon[86423]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T03:50:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:53 smithi110 ceph-mon[86423]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T03:50:54.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:53 smithi195 ceph-mon[82388]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T03:50:54.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:53 smithi195 ceph-mon[82388]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T03:50:54.840 INFO:tasks.rados.rados.0.smithi110.stdout:3853: done (0 left) 2024-09-14T03:50:54.840 INFO:tasks.rados.rados.0.smithi110.stdout:3854: read oid 36 snap 387 2024-09-14T03:50:54.840 INFO:tasks.rados.rados.0.smithi110.stdout:3854: expect (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:50:54.840 INFO:tasks.rados.rados.0.smithi110.stdout:3855: setattr oid 19 current snap is 388 2024-09-14T03:50:54.841 INFO:tasks.rados.rados.0.smithi110.stdout:3856: copy_from oid 31 from oid 11 current snap is 388 2024-09-14T03:50:54.841 INFO:tasks.rados.rados.0.smithi110.stdout:3857: snap_remove snap 384 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:3857: done (3 left) 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:3858: rmattr oid 15 current snap is 388 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:3859: rollback oid 6 current snap is 388 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 6 to 365 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 19 v 2526 (ObjNum 1164 snap 379 seq_num 1164) dirty exists 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:3856: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:3856: finishing copy_from to smithi11075562-31 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 2017 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:50:55.849 INFO:tasks.rados.rados.0.smithi110.stdout:3854: expect (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2529 (ObjNum 1161 snap 379 seq_num 1161) dirty exists 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:3859: finishing rollback tid 0 to smithi11075562-6 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:3859: finishing rollback tid 1 to smithi11075562-6 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:3859: finishing rollback tid 2 to smithi11075562-6 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2529 (ObjNum 953 snap 307 seq_num 953) dirty exists 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:3854: done (4 left) 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:3855: done (3 left) 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:3856: done (2 left) 2024-09-14T03:50:56.006 INFO:tasks.rados.rados.0.smithi110.stdout:3858: done (1 left) 2024-09-14T03:50:56.007 INFO:tasks.rados.rados.0.smithi110.stdout:3859: done (0 left) 2024-09-14T03:50:56.007 INFO:tasks.rados.rados.0.smithi110.stdout:3860: snap_remove snap 385 2024-09-14T03:50:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:55 smithi110 ceph-mon[88262]: pgmap v897: 121 pgs: 121 active+clean; 289 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 572 KiB/s wr, 1 op/s 2024-09-14T03:50:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:55 smithi110 ceph-mon[88262]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T03:50:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:55 smithi110 ceph-mon[86423]: pgmap v897: 121 pgs: 121 active+clean; 289 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 572 KiB/s wr, 1 op/s 2024-09-14T03:50:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:55 smithi110 ceph-mon[86423]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T03:50:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:55 smithi195 ceph-mon[82388]: pgmap v897: 121 pgs: 121 active+clean; 289 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 572 KiB/s wr, 1 op/s 2024-09-14T03:50:56.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:55 smithi195 ceph-mon[82388]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T03:50:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3860: done (0 left) 2024-09-14T03:50:56.851 INFO:tasks.rados.rados.0.smithi110.stdout:3861: snap_remove snap 373 2024-09-14T03:50:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:56 smithi110 ceph-mon[88262]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T03:50:57.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:56 smithi110 ceph-mon[88262]: pgmap v900: 121 pgs: 121 active+clean; 287 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 572 KiB/s wr, 2 op/s 2024-09-14T03:50:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:56 smithi110 ceph-mon[86423]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T03:50:57.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:56 smithi110 ceph-mon[86423]: pgmap v900: 121 pgs: 121 active+clean; 287 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 572 KiB/s wr, 2 op/s 2024-09-14T03:50:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:56 smithi195 ceph-mon[82388]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T03:50:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:56 smithi195 ceph-mon[82388]: pgmap v900: 121 pgs: 121 active+clean; 287 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 572 KiB/s wr, 2 op/s 2024-09-14T03:50:57.853 INFO:tasks.rados.rados.0.smithi110.stdout:3861: done (0 left) 2024-09-14T03:50:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:3862: copy_from oid 37 from oid 24 current snap is 388 2024-09-14T03:50:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:3863: copy_from oid 28 from oid 12 current snap is 388 2024-09-14T03:50:57.854 INFO:tasks.rados.rados.0.smithi110.stdout:3864: snap_remove snap 380 2024-09-14T03:50:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:57 smithi110 ceph-mon[88262]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T03:50:58.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:57 smithi110 ceph-mon[86423]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T03:50:58.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:58.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:57 smithi195 ceph-mon[82388]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T03:50:58.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:50:58.840 INFO:tasks.rados.rados.0.smithi110.stdout:3862: finishing copy_from racing read to smithi11075562-37 2024-09-14T03:50:58.840 INFO:tasks.rados.rados.0.smithi110.stdout:3862: finishing copy_from to smithi11075562-37 2024-09-14T03:50:58.840 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 2050 (ObjNum 1082 snap 342 seq_num 1082) dirty exists 2024-09-14T03:50:58.840 INFO:tasks.rados.rados.0.smithi110.stdout:3863: finishing copy_from racing read to smithi11075562-28 2024-09-14T03:50:58.840 INFO:tasks.rados.rados.0.smithi110.stdout:3863: finishing copy_from to smithi11075562-28 2024-09-14T03:50:58.840 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 28 v 2166 (ObjNum 1113 snap 354 seq_num 1113) dirty exists 2024-09-14T03:50:58.840 INFO:tasks.rados.rados.0.smithi110.stdout:3862: done (2 left) 2024-09-14T03:50:58.840 INFO:tasks.rados.rados.0.smithi110.stdout:3863: done (1 left) 2024-09-14T03:50:58.841 INFO:tasks.rados.rados.0.smithi110.stdout:3864: done (0 left) 2024-09-14T03:50:58.841 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 388 2024-09-14T03:50:58.841 INFO:tasks.rados.rados.0.smithi110.stdout:3865: seq_num 1176 ranges {5070848=499712} 2024-09-14T03:50:58.871 INFO:tasks.rados.rados.0.smithi110.stdout:3865: writing smithi11075562-16 from 5070848 to 5570560 tid 1 2024-09-14T03:50:58.871 INFO:tasks.rados.rados.0.smithi110.stdout:3866: rmattr oid 34 current snap is 388 2024-09-14T03:50:58.872 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 388 2024-09-14T03:50:58.872 INFO:tasks.rados.rados.0.smithi110.stdout:3867: seq_num 1177 ranges {5234688=516096} 2024-09-14T03:50:58.902 INFO:tasks.rados.rados.0.smithi110.stdout:3867: writing smithi11075562-21 from 5234688 to 5750784 tid 1 2024-09-14T03:50:58.903 INFO:tasks.rados.rados.0.smithi110.stdout:3868: setattr oid 37 current snap is 388 2024-09-14T03:50:58.903 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 2177 (ObjNum 1601 snap 0 seq_num 2917114208) dirty exists 2024-09-14T03:50:58.903 INFO:tasks.rados.rados.0.smithi110.stdout:3865: finishing write tid 1 to smithi11075562-16 2024-09-14T03:50:58.903 INFO:tasks.rados.rados.0.smithi110.stdout:3865: finishing write tid 2 to smithi11075562-16 2024-09-14T03:50:58.903 INFO:tasks.rados.rados.0.smithi110.stdout:3865: finishing write tid 3 to smithi11075562-16 2024-09-14T03:50:58.903 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2531 (ObjNum 1176 snap 388 seq_num 1176) dirty exists 2024-09-14T03:50:58.903 INFO:tasks.rados.rados.0.smithi110.stdout:3865: left oid 16 (ObjNum 1176 snap 388 seq_num 1176) 2024-09-14T03:50:58.904 INFO:tasks.rados.rados.0.smithi110.stdout:3865: done (3 left) 2024-09-14T03:50:58.904 INFO:tasks.rados.rados.0.smithi110.stdout:3866: done (2 left) 2024-09-14T03:50:58.904 INFO:tasks.rados.rados.0.smithi110.stdout:3869: read oid 29 snap -1 2024-09-14T03:50:58.904 INFO:tasks.rados.rados.0.smithi110.stdout:3869: expect (ObjNum 1173 snap 385 seq_num 1173) 2024-09-14T03:50:58.904 INFO:tasks.rados.rados.0.smithi110.stdout:3870: delete oid 38 current snap is 388 2024-09-14T03:50:58.906 INFO:tasks.rados.rados.0.smithi110.stdout:3867: finishing write tid 1 to smithi11075562-21 2024-09-14T03:50:58.907 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 2728 (ObjNum 1082 snap 342 seq_num 1082) dirty exists 2024-09-14T03:50:58.908 INFO:tasks.rados.rados.0.smithi110.stdout:3867: finishing write tid 2 to smithi11075562-21 2024-09-14T03:50:58.910 INFO:tasks.rados.rados.0.smithi110.stdout:3867: finishing write tid 3 to smithi11075562-21 2024-09-14T03:50:58.910 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2509 (ObjNum 1177 snap 388 seq_num 1177) dirty exists 2024-09-14T03:50:58.910 INFO:tasks.rados.rados.0.smithi110.stdout:3867: left oid 21 (ObjNum 1177 snap 388 seq_num 1177) 2024-09-14T03:50:58.911 INFO:tasks.rados.rados.0.smithi110.stdout:3867: done (3 left) 2024-09-14T03:50:58.912 INFO:tasks.rados.rados.0.smithi110.stdout:3868: done (2 left) 2024-09-14T03:50:58.912 INFO:tasks.rados.rados.0.smithi110.stdout:3870: done (1 left) 2024-09-14T03:50:58.912 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 388 2024-09-14T03:50:58.912 INFO:tasks.rados.rados.0.smithi110.stdout:3871: seq_num 1179 ranges {5570560=548864} 2024-09-14T03:50:58.944 INFO:tasks.rados.rados.0.smithi110.stdout:3871: writing smithi11075562-16 from 5570560 to 6119424 tid 1 2024-09-14T03:50:58.944 INFO:tasks.rados.rados.0.smithi110.stdout:3872: rollback oid 43 current snap is 388 2024-09-14T03:50:58.944 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 43 to 370 2024-09-14T03:50:58.944 INFO:tasks.rados.rados.0.smithi110.stdout:3869: expect (ObjNum 1173 snap 385 seq_num 1173) 2024-09-14T03:50:59.125 INFO:tasks.rados.rados.0.smithi110.stdout:3872: finishing rollback tid 0 to smithi11075562-43 2024-09-14T03:50:59.125 INFO:tasks.rados.rados.0.smithi110.stdout:3872: finishing rollback tid 1 to smithi11075562-43 2024-09-14T03:50:59.125 INFO:tasks.rados.rados.0.smithi110.stdout:3871: finishing write tid 1 to smithi11075562-16 2024-09-14T03:50:59.125 INFO:tasks.rados.rados.0.smithi110.stdout:3872: finishing rollback tid 2 to smithi11075562-43 2024-09-14T03:50:59.125 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 2512 (ObjNum 1134 snap 367 seq_num 1134) dirty exists 2024-09-14T03:50:59.125 INFO:tasks.rados.rados.0.smithi110.stdout:3871: finishing write tid 2 to smithi11075562-16 2024-09-14T03:50:59.125 INFO:tasks.rados.rados.0.smithi110.stdout:3871: finishing write tid 3 to smithi11075562-16 2024-09-14T03:50:59.126 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2533 (ObjNum 1179 snap 388 seq_num 1179) dirty exists 2024-09-14T03:50:59.126 INFO:tasks.rados.rados.0.smithi110.stdout:3871: left oid 16 (ObjNum 1179 snap 388 seq_num 1179) 2024-09-14T03:50:59.126 INFO:tasks.rados.rados.0.smithi110.stdout:3869: done (2 left) 2024-09-14T03:50:59.126 INFO:tasks.rados.rados.0.smithi110.stdout:3871: done (1 left) 2024-09-14T03:50:59.126 INFO:tasks.rados.rados.0.smithi110.stdout:3872: done (0 left) 2024-09-14T03:50:59.126 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 26 current snap is 388 2024-09-14T03:50:59.126 INFO:tasks.rados.rados.0.smithi110.stdout:3873: seq_num 1180 ranges {7995392=565248} 2024-09-14T03:50:59.169 INFO:tasks.rados.rados.0.smithi110.stdout:3873: writing smithi11075562-26 from 7995392 to 8560640 tid 1 2024-09-14T03:50:59.169 INFO:tasks.rados.rados.0.smithi110.stdout:3874: read oid 7 snap -1 2024-09-14T03:50:59.169 INFO:tasks.rados.rados.0.smithi110.stdout:3874: expect deleted 2024-09-14T03:50:59.170 INFO:tasks.rados.rados.0.smithi110.stdout:3875: snap_remove snap 375 2024-09-14T03:50:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:58 smithi110 ceph-mon[88262]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T03:50:59.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:58 smithi110 ceph-mon[88262]: pgmap v903: 121 pgs: 121 active+clean; 287 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 0 B/s wr, 6 op/s 2024-09-14T03:50:59.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:50:58 smithi110 ceph-mon[88262]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T03:50:59.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:58 smithi110 ceph-mon[86423]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T03:50:59.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:58 smithi110 ceph-mon[86423]: pgmap v903: 121 pgs: 121 active+clean; 287 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 0 B/s wr, 6 op/s 2024-09-14T03:50:59.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:50:58 smithi110 ceph-mon[86423]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T03:50:59.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:58 smithi195 ceph-mon[82388]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T03:50:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:58 smithi195 ceph-mon[82388]: pgmap v903: 121 pgs: 121 active+clean; 287 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 0 B/s wr, 6 op/s 2024-09-14T03:50:59.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:50:58 smithi195 ceph-mon[82388]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T03:50:59.870 INFO:tasks.rados.rados.0.smithi110.stdout:3873: finishing write tid 1 to smithi11075562-26 2024-09-14T03:50:59.870 INFO:tasks.rados.rados.0.smithi110.stdout:3873: finishing write tid 2 to smithi11075562-26 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3873: finishing write tid 3 to smithi11075562-26 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 26 v 2018 (ObjNum 1180 snap 388 seq_num 1180) dirty exists 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3873: left oid 26 (ObjNum 1180 snap 388 seq_num 1180) 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3873: done (2 left) 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3874: done (1 left) 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3875: done (0 left) 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3876: read oid 8 snap -1 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3876: expect deleted 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3877: copy_from oid 1 from oid 21 current snap is 388 2024-09-14T03:50:59.871 INFO:tasks.rados.rados.0.smithi110.stdout:3878: read oid 33 snap -1 2024-09-14T03:50:59.872 INFO:tasks.rados.rados.0.smithi110.stdout:3878: expect (ObjNum 1161 snap 379 seq_num 1161) 2024-09-14T03:50:59.872 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 39 current snap is 388 2024-09-14T03:50:59.872 INFO:tasks.rados.rados.0.smithi110.stdout:3879: seq_num 1181 ranges {0=581632} 2024-09-14T03:50:59.880 INFO:tasks.rados.rados.0.smithi110.stdout:3879: writing smithi11075562-39 from 0 to 581632 tid 1 2024-09-14T03:50:59.880 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 25 current snap is 388 2024-09-14T03:50:59.880 INFO:tasks.rados.rados.0.smithi110.stdout:3880: seq_num 1182 ranges {229376=606208} 2024-09-14T03:50:59.881 INFO:tasks.rados.rados.0.smithi110.stdout:3880: writing smithi11075562-25 from 229376 to 835584 tid 1 2024-09-14T03:50:59.882 INFO:tasks.rados.rados.0.smithi110.stdout:3881: read oid 36 snap -1 2024-09-14T03:50:59.882 INFO:tasks.rados.rados.0.smithi110.stdout:3881: expect (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:50:59.882 INFO:tasks.rados.rados.0.smithi110.stdout:3882: copy_from oid 11 from oid 24 current snap is 388 2024-09-14T03:50:59.882 INFO:tasks.rados.rados.0.smithi110.stdout:3876: done (6 left) 2024-09-14T03:50:59.882 INFO:tasks.rados.rados.0.smithi110.stdout:3883: read oid 14 snap -1 2024-09-14T03:50:59.882 INFO:tasks.rados.rados.0.smithi110.stdout:3883: expect (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:50:59.883 INFO:tasks.rados.rados.0.smithi110.stdout:3884: delete oid 15 current snap is 388 2024-09-14T03:50:59.888 INFO:tasks.rados.rados.0.smithi110.stdout:3884: done (7 left) 2024-09-14T03:50:59.888 INFO:tasks.rados.rados.0.smithi110.stdout:3885: snap_create 2024-09-14T03:50:59.893 INFO:tasks.rados.rados.0.smithi110.stdout:3878: expect (ObjNum 1161 snap 379 seq_num 1161) 2024-09-14T03:51:00.011 INFO:tasks.rados.rados.0.smithi110.stdout:3879: finishing write tid 1 to smithi11075562-39 2024-09-14T03:51:00.011 INFO:tasks.rados.rados.0.smithi110.stdout:3879: finishing write tid 2 to smithi11075562-39 2024-09-14T03:51:00.011 INFO:tasks.rados.rados.0.smithi110.stdout:3880: finishing write tid 1 to smithi11075562-25 2024-09-14T03:51:00.012 INFO:tasks.rados.rados.0.smithi110.stdout:3880: finishing write tid 2 to smithi11075562-25 2024-09-14T03:51:00.012 INFO:tasks.rados.rados.0.smithi110.stdout:3883: expect (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:51:00.165 INFO:tasks.rados.rados.0.smithi110.stdout:3879: finishing write tid 3 to smithi11075562-39 2024-09-14T03:51:00.166 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 39 v 2514 (ObjNum 1181 snap 388 seq_num 1181) dirty exists 2024-09-14T03:51:00.166 INFO:tasks.rados.rados.0.smithi110.stdout:3879: left oid 39 (ObjNum 1181 snap 388 seq_num 1181) 2024-09-14T03:51:00.166 INFO:tasks.rados.rados.0.smithi110.stdout:3881: expect (ObjNum 1137 snap 368 seq_num 1137) 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:3880: finishing write tid 3 to smithi11075562-25 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 25 v 2735 (ObjNum 1182 snap 388 seq_num 1182) dirty exists 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:3880: left oid 25 (ObjNum 1182 snap 388 seq_num 1182) 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:3877: finishing copy_from racing read to smithi11075562-1 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:3882: finishing copy_from racing read to smithi11075562-11 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:3877: finishing copy_from to smithi11075562-1 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2509 (ObjNum 1177 snap 388 seq_num 1177) dirty exists 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:3882: finishing copy_from to smithi11075562-11 2024-09-14T03:51:00.319 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 2050 (ObjNum 1082 snap 342 seq_num 1082) dirty exists 2024-09-14T03:51:00.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:51:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:51:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:00.876 INFO:tasks.rados.rados.0.smithi110.stdout:3877: done (7 left) 2024-09-14T03:51:00.876 INFO:tasks.rados.rados.0.smithi110.stdout:3878: done (6 left) 2024-09-14T03:51:00.876 INFO:tasks.rados.rados.0.smithi110.stdout:3879: done (5 left) 2024-09-14T03:51:00.877 INFO:tasks.rados.rados.0.smithi110.stdout:3880: done (4 left) 2024-09-14T03:51:00.877 INFO:tasks.rados.rados.0.smithi110.stdout:3881: done (3 left) 2024-09-14T03:51:00.877 INFO:tasks.rados.rados.0.smithi110.stdout:3882: done (2 left) 2024-09-14T03:51:00.877 INFO:tasks.rados.rados.0.smithi110.stdout:3883: done (1 left) 2024-09-14T03:51:00.877 INFO:tasks.rados.rados.0.smithi110.stdout:3885: done (0 left) 2024-09-14T03:51:00.877 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 389 2024-09-14T03:51:00.877 INFO:tasks.rados.rados.0.smithi110.stdout:3886: seq_num 1184 ranges {0=638976} 2024-09-14T03:51:00.879 INFO:tasks.rados.rados.0.smithi110.stdout:3886: writing smithi11075562-45 from 0 to 638976 tid 1 2024-09-14T03:51:00.879 INFO:tasks.rados.rados.0.smithi110.stdout:3887: snap_remove snap 365 2024-09-14T03:51:01.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:00 smithi110 ceph-mon[86423]: pgmap v905: 121 pgs: 121 active+clean; 302 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 3.7 MiB/s wr, 9 op/s 2024-09-14T03:51:01.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:00 smithi110 ceph-mon[86423]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T03:51:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:00 smithi110 ceph-mon[88262]: pgmap v905: 121 pgs: 121 active+clean; 302 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 3.7 MiB/s wr, 9 op/s 2024-09-14T03:51:01.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:00 smithi110 ceph-mon[88262]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T03:51:01.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:00 smithi195 ceph-mon[82388]: pgmap v905: 121 pgs: 121 active+clean; 302 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 3.7 MiB/s wr, 9 op/s 2024-09-14T03:51:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:00 smithi195 ceph-mon[82388]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T03:51:01.878 INFO:tasks.rados.rados.0.smithi110.stdout:3887: done (1 left) 2024-09-14T03:51:01.878 INFO:tasks.rados.rados.0.smithi110.stdout:3888: snap_remove snap 369 2024-09-14T03:51:01.880 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:51:01] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.51.0" 2024-09-14T03:51:02.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[88262]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T03:51:02.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:02.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:02.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:02.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:51:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[86423]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T03:51:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:02.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:02.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:02.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:01 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:51:02.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:01 smithi195 ceph-mon[82388]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T03:51:02.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:01 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:02.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:01 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:51:02.882 INFO:tasks.rados.rados.0.smithi110.stdout:3888: done (1 left) 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:3889: copy_from oid 30 from oid 2 current snap is 389 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:3890: copy_from oid 46 from oid 50 current snap is 389 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:3891: setattr oid 15 current snap is 389 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:3892: delete oid 31 current snap is 389 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:3886: finishing write tid 1 to smithi11075562-45 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:3886: finishing write tid 2 to smithi11075562-45 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:3886: finishing write tid 3 to smithi11075562-45 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2743 (ObjNum 1184 snap 389 seq_num 1184) dirty exists 2024-09-14T03:51:02.883 INFO:tasks.rados.rados.0.smithi110.stdout:3886: left oid 45 (ObjNum 1184 snap 389 seq_num 1184) 2024-09-14T03:51:02.895 INFO:tasks.rados.rados.0.smithi110.stdout:3886: done (4 left) 2024-09-14T03:51:02.895 INFO:tasks.rados.rados.0.smithi110.stdout:3892: done (3 left) 2024-09-14T03:51:02.895 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 389 2024-09-14T03:51:02.896 INFO:tasks.rados.rados.0.smithi110.stdout:3893: seq_num 1186 ranges {8249344=671744} 2024-09-14T03:51:02.944 INFO:tasks.rados.rados.0.smithi110.stdout:3893: writing smithi11075562-41 from 8249344 to 8921088 tid 1 2024-09-14T03:51:02.944 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 389 2024-09-14T03:51:02.944 INFO:tasks.rados.rados.0.smithi110.stdout:3894: seq_num 1187 ranges {6365184=688128} 2024-09-14T03:51:02.981 INFO:tasks.rados.rados.0.smithi110.stdout:3894: writing smithi11075562-3 from 6365184 to 7053312 tid 1 2024-09-14T03:51:02.981 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 29 current snap is 389 2024-09-14T03:51:02.981 INFO:tasks.rados.rados.0.smithi110.stdout:3895: seq_num 1188 ranges {3932160=704512} 2024-09-14T03:51:03.006 INFO:tasks.rados.rados.0.smithi110.stdout:3895: writing smithi11075562-29 from 3932160 to 4636672 tid 1 2024-09-14T03:51:03.006 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 12 current snap is 389 2024-09-14T03:51:03.006 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2531 (ObjNum 65 snap 0 seq_num 1684821852) dirty exists 2024-09-14T03:51:03.006 INFO:tasks.rados.rados.0.smithi110.stdout:3889: finishing copy_from racing read to smithi11075562-30 2024-09-14T03:51:03.006 INFO:tasks.rados.rados.0.smithi110.stdout:3889: finishing copy_from to smithi11075562-30 2024-09-14T03:51:03.006 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 2524 (ObjNum 1066 snap 336 seq_num 1066) dirty exists 2024-09-14T03:51:03.006 INFO:tasks.rados.rados.0.smithi110.stdout:3890: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:51:03.006 INFO:tasks.rados.rados.0.smithi110.stdout:3896: seq_num 1189 ranges {12967936=720896} 2024-09-14T03:51:03.078 INFO:tasks.rados.rados.0.smithi110.stdout:3896: writing smithi11075562-12 from 12967936 to 13688832 tid 1 2024-09-14T03:51:03.078 INFO:tasks.rados.rados.0.smithi110.stdout:3889: done (6 left) 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3891: done (5 left) 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3897: snap_create 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3890: finishing copy_from to smithi11075562-46 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 2047 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3893: finishing write tid 1 to smithi11075562-41 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3893: finishing write tid 2 to smithi11075562-41 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3893: finishing write tid 3 to smithi11075562-41 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 2516 (ObjNum 1186 snap 389 seq_num 1186) dirty exists 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3893: left oid 41 (ObjNum 1186 snap 389 seq_num 1186) 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3894: finishing write tid 1 to smithi11075562-3 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3894: finishing write tid 2 to smithi11075562-3 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3894: finishing write tid 3 to smithi11075562-3 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 2526 (ObjNum 1187 snap 389 seq_num 1187) dirty exists 2024-09-14T03:51:03.079 INFO:tasks.rados.rados.0.smithi110.stdout:3894: left oid 3 (ObjNum 1187 snap 389 seq_num 1187) 2024-09-14T03:51:03.080 INFO:tasks.rados.rados.0.smithi110.stdout:3895: finishing write tid 1 to smithi11075562-29 2024-09-14T03:51:03.080 INFO:tasks.rados.rados.0.smithi110.stdout:3895: finishing write tid 2 to smithi11075562-29 2024-09-14T03:51:03.080 INFO:tasks.rados.rados.0.smithi110.stdout:3895: finishing write tid 3 to smithi11075562-29 2024-09-14T03:51:03.080 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 29 v 1659 (ObjNum 1188 snap 389 seq_num 1188) dirty exists 2024-09-14T03:51:03.080 INFO:tasks.rados.rados.0.smithi110.stdout:3895: left oid 29 (ObjNum 1188 snap 389 seq_num 1188) 2024-09-14T03:51:03.084 INFO:tasks.rados.rados.0.smithi110.stdout:3896: finishing write tid 1 to smithi11075562-12 2024-09-14T03:51:03.084 INFO:tasks.rados.rados.0.smithi110.stdout:3896: finishing write tid 2 to smithi11075562-12 2024-09-14T03:51:03.086 INFO:tasks.rados.rados.0.smithi110.stdout:3896: finishing write tid 3 to smithi11075562-12 2024-09-14T03:51:03.086 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2770 (ObjNum 1189 snap 389 seq_num 1189) dirty exists 2024-09-14T03:51:03.086 INFO:tasks.rados.rados.0.smithi110.stdout:3896: left oid 12 (ObjNum 1189 snap 389 seq_num 1189) 2024-09-14T03:51:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:02 smithi110 ceph-mon[88262]: pgmap v908: 121 pgs: 121 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.7 MiB/s wr, 14 op/s 2024-09-14T03:51:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:02 smithi110 ceph-mon[88262]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T03:51:03.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:02 smithi110 ceph-mon[88262]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T03:51:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:02 smithi110 ceph-mon[86423]: pgmap v908: 121 pgs: 121 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.7 MiB/s wr, 14 op/s 2024-09-14T03:51:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:02 smithi110 ceph-mon[86423]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T03:51:03.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:02 smithi110 ceph-mon[86423]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T03:51:03.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:02 smithi195 ceph-mon[82388]: pgmap v908: 121 pgs: 121 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.7 MiB/s wr, 14 op/s 2024-09-14T03:51:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:02 smithi195 ceph-mon[82388]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T03:51:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:02 smithi195 ceph-mon[82388]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T03:51:03.841 INFO:tasks.rados.rados.0.smithi110.stdout:3890: done (5 left) 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3893: done (4 left) 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3894: done (3 left) 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3895: done (2 left) 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3896: done (1 left) 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3897: done (0 left) 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3898: read oid 22 snap -1 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3898: expect (ObjNum 1102 snap 349 seq_num 1102) 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3899: copy_from oid 36 from oid 35 current snap is 390 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3900: read oid 23 snap -1 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3900: expect (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:51:03.842 INFO:tasks.rados.rados.0.smithi110.stdout:3901: copy_from oid 32 from oid 31 current snap is 390 2024-09-14T03:51:03.843 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 7 current snap is 390 2024-09-14T03:51:03.843 INFO:tasks.rados.rados.0.smithi110.stdout:3902: seq_num 1190 ranges {0=737280} 2024-09-14T03:51:03.846 INFO:tasks.rados.rados.0.smithi110.stdout:3902: writing smithi11075562-7 from 0 to 737280 tid 1 2024-09-14T03:51:03.846 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 43 current snap is 390 2024-09-14T03:51:03.846 INFO:tasks.rados.rados.0.smithi110.stdout:3901: finishing copy_from to smithi11075562-32 2024-09-14T03:51:03.847 INFO:tasks.rados.rados.0.smithi110.stdout:3901: got expected ENOENT (src dne) 2024-09-14T03:51:03.847 INFO:tasks.rados.rados.0.smithi110.stdout:3901: finishing copy_from racing read to smithi11075562-32 2024-09-14T03:51:03.847 INFO:tasks.rados.rados.0.smithi110.stdout:3903: seq_num 1191 ranges {3653632=753664} 2024-09-14T03:51:03.872 INFO:tasks.rados.rados.0.smithi110.stdout:3903: writing smithi11075562-43 from 3653632 to 4407296 tid 1 2024-09-14T03:51:03.872 INFO:tasks.rados.rados.0.smithi110.stdout:3901: done (5 left) 2024-09-14T03:51:03.872 INFO:tasks.rados.rados.0.smithi110.stdout:3904: rollback oid 24 current snap is 390 2024-09-14T03:51:03.872 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 24 to 389 2024-09-14T03:51:03.872 INFO:tasks.rados.rados.0.smithi110.stdout:3898: expect (ObjNum 1102 snap 349 seq_num 1102) 2024-09-14T03:51:03.923 INFO:tasks.rados.rados.0.smithi110.stdout:3902: finishing write tid 1 to smithi11075562-7 2024-09-14T03:51:03.923 INFO:tasks.rados.rados.0.smithi110.stdout:3902: finishing write tid 2 to smithi11075562-7 2024-09-14T03:51:03.923 INFO:tasks.rados.rados.0.smithi110.stdout:3899: finishing copy_from racing read to smithi11075562-36 2024-09-14T03:51:03.923 INFO:tasks.rados.rados.0.smithi110.stdout:3902: finishing write tid 3 to smithi11075562-7 2024-09-14T03:51:03.923 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 1661 (ObjNum 1190 snap 390 seq_num 1190) dirty exists 2024-09-14T03:51:03.923 INFO:tasks.rados.rados.0.smithi110.stdout:3902: left oid 7 (ObjNum 1190 snap 390 seq_num 1190) 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3899: finishing copy_from to smithi11075562-36 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2267 (ObjNum 1157 snap 378 seq_num 1157) dirty exists 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3898: done (5 left) 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3899: done (4 left) 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3902: done (3 left) 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3905: copy_from oid 5 from oid 16 current snap is 390 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3903: finishing write tid 1 to smithi11075562-43 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3903: finishing write tid 2 to smithi11075562-43 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3903: finishing write tid 3 to smithi11075562-43 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 2518 (ObjNum 1191 snap 390 seq_num 1191) dirty exists 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3903: left oid 43 (ObjNum 1191 snap 390 seq_num 1191) 2024-09-14T03:51:03.924 INFO:tasks.rados.rados.0.smithi110.stdout:3900: expect (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3904: finishing rollback tid 0 to smithi11075562-24 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3904: finishing rollback tid 1 to smithi11075562-24 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3904: finishing rollback tid 2 to smithi11075562-24 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 24 v 2171 (ObjNum 1082 snap 342 seq_num 1082) dirty exists 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3905: finishing copy_from racing read to smithi11075562-5 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3905: finishing copy_from to smithi11075562-5 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 5 v 2533 (ObjNum 1179 snap 388 seq_num 1179) dirty exists 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3900: done (3 left) 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3903: done (2 left) 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3904: done (1 left) 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3905: done (0 left) 2024-09-14T03:51:05.070 INFO:tasks.rados.rados.0.smithi110.stdout:3906: snap_create 2024-09-14T03:51:05.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:04 smithi110 ceph-mon[88262]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T03:51:05.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:04 smithi110 ceph-mon[86423]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T03:51:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:04 smithi195 ceph-mon[82388]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T03:51:05.851 INFO:tasks.rados.rados.0.smithi110.stdout:3906: done (0 left) 2024-09-14T03:51:05.851 INFO:tasks.rados.rados.0.smithi110.stdout:3907: rmattr oid 4 current snap is 391 2024-09-14T03:51:05.851 INFO:tasks.rados.rados.0.smithi110.stdout:3908: read oid 37 snap -1 2024-09-14T03:51:05.851 INFO:tasks.rados.rados.0.smithi110.stdout:3908: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:51:05.851 INFO:tasks.rados.rados.0.smithi110.stdout:3909: snap_remove snap 386 2024-09-14T03:51:06.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:05 smithi110 ceph-mon[88262]: pgmap v912: 121 pgs: 121 active+clean; 321 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 4.3 MiB/s wr, 15 op/s 2024-09-14T03:51:06.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:05 smithi110 ceph-mon[86423]: pgmap v912: 121 pgs: 121 active+clean; 321 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 4.3 MiB/s wr, 15 op/s 2024-09-14T03:51:06.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:05 smithi195 ceph-mon[82388]: pgmap v912: 121 pgs: 121 active+clean; 321 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 4.3 MiB/s wr, 15 op/s 2024-09-14T03:51:06.858 INFO:tasks.rados.rados.0.smithi110.stdout:3909: done (2 left) 2024-09-14T03:51:06.858 INFO:tasks.rados.rados.0.smithi110.stdout:3910: rmattr oid 8 current snap is 391 2024-09-14T03:51:06.858 INFO:tasks.rados.rados.0.smithi110.stdout:3910: done (2 left) 2024-09-14T03:51:06.858 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 6 current snap is 391 2024-09-14T03:51:06.858 INFO:tasks.rados.rados.0.smithi110.stdout:3911: seq_num 1192 ranges {8306688=770048} 2024-09-14T03:51:06.908 INFO:tasks.rados.rados.0.smithi110.stdout:3911: writing smithi11075562-6 from 8306688 to 9076736 tid 1 2024-09-14T03:51:06.908 INFO:tasks.rados.rados.0.smithi110.stdout:3912: read oid 15 snap -1 2024-09-14T03:51:06.908 INFO:tasks.rados.rados.0.smithi110.stdout:3912: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:51:06.908 INFO:tasks.rados.rados.0.smithi110.stdout:3913: read oid 41 snap -1 2024-09-14T03:51:06.908 INFO:tasks.rados.rados.0.smithi110.stdout:3913: expect (ObjNum 1186 snap 389 seq_num 1186) 2024-09-14T03:51:06.908 INFO:tasks.rados.rados.0.smithi110.stdout:3914: copy_from oid 33 from oid 42 current snap is 391 2024-09-14T03:51:06.908 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 4 v 2179 (ObjNum 1172 snap 382 seq_num 1172) dirty exists 2024-09-14T03:51:06.908 INFO:tasks.rados.rados.0.smithi110.stdout:3908: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:51:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:06 smithi110 ceph-mon[88262]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T03:51:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:06 smithi110 ceph-mon[88262]: pgmap v914: 121 pgs: 121 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 4.8 MiB/s wr, 17 op/s 2024-09-14T03:51:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:06 smithi110 ceph-mon[86423]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T03:51:07.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:06 smithi110 ceph-mon[86423]: pgmap v914: 121 pgs: 121 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 4.8 MiB/s wr, 17 op/s 2024-09-14T03:51:07.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:06 smithi195 ceph-mon[82388]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T03:51:07.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:06 smithi195 ceph-mon[82388]: pgmap v914: 121 pgs: 121 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 4.8 MiB/s wr, 17 op/s 2024-09-14T03:51:07.627 INFO:tasks.rados.rados.0.smithi110.stdout:3912: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3911: finishing write tid 1 to smithi11075562-6 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3911: finishing write tid 2 to smithi11075562-6 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3911: finishing write tid 3 to smithi11075562-6 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 6 v 2535 (ObjNum 1192 snap 391 seq_num 1192) dirty exists 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3911: left oid 6 (ObjNum 1192 snap 391 seq_num 1192) 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3914: finishing copy_from racing read to smithi11075562-33 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3907: done (5 left) 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3908: done (4 left) 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3911: done (3 left) 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3912: done (2 left) 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 391 2024-09-14T03:51:07.631 INFO:tasks.rados.rados.0.smithi110.stdout:3913: expect (ObjNum 1186 snap 389 seq_num 1186) 2024-09-14T03:51:08.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:07 smithi110 ceph-mon[88262]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T03:51:08.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:07 smithi110 ceph-mon[86423]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T03:51:08.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:07 smithi195 ceph-mon[82388]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T03:51:08.357 INFO:tasks.rados.rados.0.smithi110.stdout:3914: finishing copy_from to smithi11075562-33 2024-09-14T03:51:08.357 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 33 v 2255 (ObjNum 1144 snap 373 seq_num 1144) dirty exists 2024-09-14T03:51:08.357 INFO:tasks.rados.rados.0.smithi110.stdout:3915: seq_num 1193 ranges {0=786432} 2024-09-14T03:51:08.361 INFO:tasks.rados.rados.0.smithi110.stdout:3915: writing smithi11075562-27 from 0 to 786432 tid 1 2024-09-14T03:51:08.361 INFO:tasks.rados.rados.0.smithi110.stdout:3913: done (2 left) 2024-09-14T03:51:08.361 INFO:tasks.rados.rados.0.smithi110.stdout:3914: done (1 left) 2024-09-14T03:51:08.362 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 16 current snap is 391 2024-09-14T03:51:08.362 INFO:tasks.rados.rados.0.smithi110.stdout:3916: seq_num 1194 ranges {6119424=802816} 2024-09-14T03:51:08.398 INFO:tasks.rados.rados.0.smithi110.stdout:3916: writing smithi11075562-16 from 6119424 to 6922240 tid 1 2024-09-14T03:51:08.398 INFO:tasks.rados.rados.0.smithi110.stdout:3917: snap_remove snap 390 2024-09-14T03:51:08.843 INFO:tasks.rados.rados.0.smithi110.stdout:3915: finishing write tid 1 to smithi11075562-27 2024-09-14T03:51:08.843 INFO:tasks.rados.rados.0.smithi110.stdout:3915: finishing write tid 2 to smithi11075562-27 2024-09-14T03:51:08.843 INFO:tasks.rados.rados.0.smithi110.stdout:3915: finishing write tid 3 to smithi11075562-27 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1820 (ObjNum 1193 snap 391 seq_num 1193) dirty exists 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3915: left oid 27 (ObjNum 1193 snap 391 seq_num 1193) 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3916: finishing write tid 1 to smithi11075562-16 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3916: finishing write tid 2 to smithi11075562-16 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3916: finishing write tid 3 to smithi11075562-16 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 16 v 2537 (ObjNum 1194 snap 391 seq_num 1194) dirty exists 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3916: left oid 16 (ObjNum 1194 snap 391 seq_num 1194) 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3915: done (2 left) 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3916: done (1 left) 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3917: done (0 left) 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 41 current snap is 391 2024-09-14T03:51:08.844 INFO:tasks.rados.rados.0.smithi110.stdout:3918: seq_num 1195 ranges {8921088=417792,9338880=401408} 2024-09-14T03:51:08.893 INFO:tasks.rados.rados.0.smithi110.stdout:3918: writing smithi11075562-41 from 8921088 to 9338880 tid 1 2024-09-14T03:51:08.896 INFO:tasks.rados.rados.0.smithi110.stdout:3918: writing smithi11075562-41 from 9338880 to 9740288 tid 2 2024-09-14T03:51:08.896 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 22 current snap is 391 2024-09-14T03:51:08.896 INFO:tasks.rados.rados.0.smithi110.stdout:3919: seq_num 1196 ranges {1662976=434176,2097152=401408} 2024-09-14T03:51:08.908 INFO:tasks.rados.rados.0.smithi110.stdout:3919: writing smithi11075562-22 from 1662976 to 2097152 tid 1 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3919: writing smithi11075562-22 from 2097152 to 2498560 tid 2 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3920: rollback oid 11 current snap is 391 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 11 to 377 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3918: finishing write tid 1 to smithi11075562-41 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3918: finishing write tid 2 to smithi11075562-41 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3918: finishing write tid 3 to smithi11075562-41 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3918: finishing write tid 4 to smithi11075562-41 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 41 v 2521 (ObjNum 1195 snap 391 seq_num 1195) dirty exists 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3918: left oid 41 (ObjNum 1195 snap 391 seq_num 1195) 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3918: done (2 left) 2024-09-14T03:51:08.911 INFO:tasks.rados.rados.0.smithi110.stdout:3921: read oid 30 snap -1 2024-09-14T03:51:08.912 INFO:tasks.rados.rados.0.smithi110.stdout:3921: expect (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:51:08.912 INFO:tasks.rados.rados.0.smithi110.stdout:3922: read oid 10 snap -1 2024-09-14T03:51:08.912 INFO:tasks.rados.rados.0.smithi110.stdout:3922: expect (ObjNum 1170 snap 381 seq_num 1170) 2024-09-14T03:51:08.912 INFO:tasks.rados.rados.0.smithi110.stdout:3923: rollback oid 38 current snap is 391 2024-09-14T03:51:08.912 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 38 to 383 2024-09-14T03:51:08.912 INFO:tasks.rados.rados.0.smithi110.stdout:3924: rollback oid 12 current snap is 391 2024-09-14T03:51:08.912 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 12 to 383 2024-09-14T03:51:08.912 INFO:tasks.rados.rados.0.smithi110.stdout:3925: delete oid 1 current snap is 391 2024-09-14T03:51:08.915 INFO:tasks.rados.rados.0.smithi110.stdout:3919: finishing write tid 1 to smithi11075562-22 2024-09-14T03:51:08.915 INFO:tasks.rados.rados.0.smithi110.stdout:3923: finishing rollback tid 1 to smithi11075562-38 2024-09-14T03:51:08.917 INFO:tasks.rados.rados.0.smithi110.stdout:3920: finishing rollback tid 0 to smithi11075562-11 2024-09-14T03:51:08.918 INFO:tasks.rados.rados.0.smithi110.stdout:3925: done (6 left) 2024-09-14T03:51:08.918 INFO:tasks.rados.rados.0.smithi110.stdout:3926: rmattr oid 29 current snap is 391 2024-09-14T03:51:08.918 INFO:tasks.rados.rados.0.smithi110.stdout:3926: done (6 left) 2024-09-14T03:51:08.918 INFO:tasks.rados.rados.0.smithi110.stdout:3927: rollback oid 49 current snap is 391 2024-09-14T03:51:08.918 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 370 2024-09-14T03:51:08.918 INFO:tasks.rados.rados.0.smithi110.stdout:3928: snap_remove snap 388 2024-09-14T03:51:09.860 INFO:tasks.rados.rados.0.smithi110.stdout:3928: done (7 left) 2024-09-14T03:51:09.860 INFO:tasks.rados.rados.0.smithi110.stdout:3929: delete oid 35 current snap is 391 2024-09-14T03:51:09.860 INFO:tasks.rados.rados.0.smithi110.stdout:3923: finishing rollback tid 2 to smithi11075562-38 2024-09-14T03:51:09.860 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 38 v 1663 (ObjNum 1127 snap 366 seq_num 1127) dirty exists 2024-09-14T03:51:09.860 INFO:tasks.rados.rados.0.smithi110.stdout:3920: finishing rollback tid 1 to smithi11075562-11 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3924: finishing rollback tid 0 to smithi11075562-12 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3919: finishing write tid 2 to smithi11075562-22 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3919: finishing write tid 3 to smithi11075562-22 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3920: finishing rollback tid 2 to smithi11075562-11 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 11 v 2053 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3924: finishing rollback tid 1 to smithi11075562-12 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3924: finishing rollback tid 2 to smithi11075562-12 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 12 v 2786 (ObjNum 1113 snap 354 seq_num 1113) dirty exists 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3927: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3927: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2512 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:51:09.861 INFO:tasks.rados.rados.0.smithi110.stdout:3921: expect (ObjNum 1066 snap 336 seq_num 1066) 2024-09-14T03:51:09.892 INFO:tasks.rados.rados.0.smithi110.stdout:3919: finishing write tid 4 to smithi11075562-22 2024-09-14T03:51:09.892 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2534 (ObjNum 1196 snap 391 seq_num 1196) dirty exists 2024-09-14T03:51:09.892 INFO:tasks.rados.rados.0.smithi110.stdout:3919: left oid 22 (ObjNum 1196 snap 391 seq_num 1196) 2024-09-14T03:51:09.892 INFO:tasks.rados.rados.0.smithi110.stdout:3922: expect (ObjNum 1170 snap 381 seq_num 1170) 2024-09-14T03:51:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:09 smithi110 ceph-mon[88262]: pgmap v916: 121 pgs: 121 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 4.2 MiB/s wr, 14 op/s 2024-09-14T03:51:10.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:09 smithi110 ceph-mon[88262]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T03:51:10.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:09 smithi110 ceph-mon[86423]: pgmap v916: 121 pgs: 121 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 4.2 MiB/s wr, 14 op/s 2024-09-14T03:51:10.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:09 smithi110 ceph-mon[86423]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T03:51:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:09 smithi195 ceph-mon[82388]: pgmap v916: 121 pgs: 121 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 4.2 MiB/s wr, 14 op/s 2024-09-14T03:51:10.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:09 smithi195 ceph-mon[82388]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T03:51:10.447 INFO:tasks.rados.rados.0.smithi110.stdout:3919: done (7 left) 2024-09-14T03:51:10.447 INFO:tasks.rados.rados.0.smithi110.stdout:3920: done (6 left) 2024-09-14T03:51:10.448 INFO:tasks.rados.rados.0.smithi110.stdout:3921: done (5 left) 2024-09-14T03:51:10.448 INFO:tasks.rados.rados.0.smithi110.stdout:3922: done (4 left) 2024-09-14T03:51:10.448 INFO:tasks.rados.rados.0.smithi110.stdout:3923: done (3 left) 2024-09-14T03:51:10.448 INFO:tasks.rados.rados.0.smithi110.stdout:3924: done (2 left) 2024-09-14T03:51:10.448 INFO:tasks.rados.rados.0.smithi110.stdout:3927: done (1 left) 2024-09-14T03:51:10.448 INFO:tasks.rados.rados.0.smithi110.stdout:3929: done (0 left) 2024-09-14T03:51:10.448 INFO:tasks.rados.rados.0.smithi110.stdout:3930: snap_remove snap 389 2024-09-14T03:51:10.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:51:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:51:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:10.866 INFO:tasks.rados.rados.0.smithi110.stdout:3930: done (0 left) 2024-09-14T03:51:10.866 INFO:tasks.rados.rados.0.smithi110.stdout:3931: snap_remove snap 381 2024-09-14T03:51:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:10 smithi110 ceph-mon[88262]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T03:51:11.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:10 smithi110 ceph-mon[88262]: pgmap v919: 121 pgs: 121 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 13 op/s 2024-09-14T03:51:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:10 smithi110 ceph-mon[86423]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T03:51:11.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:10 smithi110 ceph-mon[86423]: pgmap v919: 121 pgs: 121 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 13 op/s 2024-09-14T03:51:11.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:10 smithi195 ceph-mon[82388]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T03:51:11.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:10 smithi195 ceph-mon[82388]: pgmap v919: 121 pgs: 121 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 13 op/s 2024-09-14T03:51:11.867 INFO:tasks.rados.rados.0.smithi110.stdout:3931: done (0 left) 2024-09-14T03:51:11.867 INFO:tasks.rados.rados.0.smithi110.stdout:3932: rmattr oid 1 current snap is 391 2024-09-14T03:51:11.867 INFO:tasks.rados.rados.0.smithi110.stdout:3932: done (0 left) 2024-09-14T03:51:11.867 INFO:tasks.rados.rados.0.smithi110.stdout:3933: snap_remove snap 382 2024-09-14T03:51:11.870 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:51:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:51:11] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.51.0" 2024-09-14T03:51:12.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:11 smithi110 ceph-mon[86423]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T03:51:12.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:11 smithi110 ceph-mon[88262]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T03:51:12.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:11 smithi195 ceph-mon[82388]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T03:51:12.868 INFO:tasks.rados.rados.0.smithi110.stdout:3933: done (0 left) 2024-09-14T03:51:12.868 INFO:tasks.rados.rados.0.smithi110.stdout:3934: delete oid 18 current snap is 391 2024-09-14T03:51:12.882 INFO:tasks.rados.rados.0.smithi110.stdout:3934: done (0 left) 2024-09-14T03:51:12.882 INFO:tasks.rados.rados.0.smithi110.stdout:3935: setattr oid 31 current snap is 391 2024-09-14T03:51:12.885 INFO:tasks.rados.rados.0.smithi110.stdout:3936: copy_from oid 2 from oid 7 current snap is 391 2024-09-14T03:51:12.885 INFO:tasks.rados.rados.0.smithi110.stdout:3937: read oid 50 snap -1 2024-09-14T03:51:12.886 INFO:tasks.rados.rados.0.smithi110.stdout:3937: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:51:12.886 INFO:tasks.rados.rados.0.smithi110.stdout:3938: read oid 6 snap -1 2024-09-14T03:51:12.886 INFO:tasks.rados.rados.0.smithi110.stdout:3938: expect (ObjNum 1192 snap 391 seq_num 1192) 2024-09-14T03:51:12.886 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 8 current snap is 391 2024-09-14T03:51:12.886 INFO:tasks.rados.rados.0.smithi110.stdout:3939: seq_num 1200 ranges {0=499712,499712=401408} 2024-09-14T03:51:12.889 INFO:tasks.rados.rados.0.smithi110.stdout:3939: writing smithi11075562-8 from 0 to 499712 tid 1 2024-09-14T03:51:12.892 INFO:tasks.rados.rados.0.smithi110.stdout:3939: writing smithi11075562-8 from 499712 to 901120 tid 2 2024-09-14T03:51:12.892 INFO:tasks.rados.rados.0.smithi110.stdout:3940: snap_remove snap 377 2024-09-14T03:51:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:12 smithi110 ceph-mon[88262]: pgmap v921: 121 pgs: 121 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 13 op/s 2024-09-14T03:51:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:12 smithi110 ceph-mon[88262]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T03:51:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:13.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:12 smithi110 ceph-mon[88262]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T03:51:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:12 smithi110 ceph-mon[86423]: pgmap v921: 121 pgs: 121 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 13 op/s 2024-09-14T03:51:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:12 smithi110 ceph-mon[86423]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T03:51:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:13.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:12 smithi110 ceph-mon[86423]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T03:51:13.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:12 smithi195 ceph-mon[82388]: pgmap v921: 121 pgs: 121 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 13 op/s 2024-09-14T03:51:13.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:12 smithi195 ceph-mon[82388]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T03:51:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:13.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:12 smithi195 ceph-mon[82388]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T03:51:13.846 INFO:tasks.rados.rados.0.smithi110.stdout:3940: done (5 left) 2024-09-14T03:51:13.846 INFO:tasks.rados.rados.0.smithi110.stdout:3941: delete oid 21 current snap is 391 2024-09-14T03:51:13.846 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 2180 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-14T03:51:13.847 INFO:tasks.rados.rados.0.smithi110.stdout:3936: finishing copy_from racing read to smithi11075562-2 2024-09-14T03:51:13.847 INFO:tasks.rados.rados.0.smithi110.stdout:3936: finishing copy_from to smithi11075562-2 2024-09-14T03:51:13.847 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 2 v 1661 (ObjNum 1190 snap 390 seq_num 1190) dirty exists 2024-09-14T03:51:13.847 INFO:tasks.rados.rados.0.smithi110.stdout:3939: finishing write tid 1 to smithi11075562-8 2024-09-14T03:51:13.847 INFO:tasks.rados.rados.0.smithi110.stdout:3939: finishing write tid 2 to smithi11075562-8 2024-09-14T03:51:13.847 INFO:tasks.rados.rados.0.smithi110.stdout:3937: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:51:14.300 INFO:tasks.rados.rados.0.smithi110.stdout:3939: finishing write tid 3 to smithi11075562-8 2024-09-14T03:51:14.300 INFO:tasks.rados.rados.0.smithi110.stdout:3939: finishing write tid 4 to smithi11075562-8 2024-09-14T03:51:14.300 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 8 v 2537 (ObjNum 1200 snap 391 seq_num 1200) dirty exists 2024-09-14T03:51:14.300 INFO:tasks.rados.rados.0.smithi110.stdout:3939: left oid 8 (ObjNum 1200 snap 391 seq_num 1200) 2024-09-14T03:51:14.300 INFO:tasks.rados.rados.0.smithi110.stdout:3938: expect (ObjNum 1192 snap 391 seq_num 1192) 2024-09-14T03:51:14.986 INFO:tasks.rados.rados.0.smithi110.stdout:3935: done (5 left) 2024-09-14T03:51:14.986 INFO:tasks.rados.rados.0.smithi110.stdout:3936: done (4 left) 2024-09-14T03:51:14.986 INFO:tasks.rados.rados.0.smithi110.stdout:3937: done (3 left) 2024-09-14T03:51:14.986 INFO:tasks.rados.rados.0.smithi110.stdout:3938: done (2 left) 2024-09-14T03:51:14.986 INFO:tasks.rados.rados.0.smithi110.stdout:3939: done (1 left) 2024-09-14T03:51:14.986 INFO:tasks.rados.rados.0.smithi110.stdout:3941: done (0 left) 2024-09-14T03:51:14.986 INFO:tasks.rados.rados.0.smithi110.stdout:3942: setattr oid 34 current snap is 391 2024-09-14T03:51:14.987 INFO:tasks.rados.rados.0.smithi110.stdout:3943: snap_create 2024-09-14T03:51:14.989 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 2181 (ObjNum 129 snap 0 seq_num 64500384) dirty exists 2024-09-14T03:51:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:14 smithi110 ceph-mon[88262]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T03:51:15.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:14 smithi110 ceph-mon[86423]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T03:51:15.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:14 smithi195 ceph-mon[82388]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T03:51:15.853 INFO:tasks.rados.rados.0.smithi110.stdout:3942: done (1 left) 2024-09-14T03:51:15.853 INFO:tasks.rados.rados.0.smithi110.stdout:3943: done (0 left) 2024-09-14T03:51:15.853 INFO:tasks.rados.rados.0.smithi110.stdout:3944: copy_from oid 15 from oid 17 current snap is 392 2024-09-14T03:51:15.853 INFO:tasks.rados.rados.0.smithi110.stdout:3945: snap_remove snap 383 2024-09-14T03:51:16.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:15 smithi110 ceph-mon[88262]: pgmap v925: 121 pgs: 121 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 584 KiB/s wr, 5 op/s 2024-09-14T03:51:16.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:15 smithi110 ceph-mon[86423]: pgmap v925: 121 pgs: 121 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 584 KiB/s wr, 5 op/s 2024-09-14T03:51:16.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:15 smithi195 ceph-mon[82388]: pgmap v925: 121 pgs: 121 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 584 KiB/s wr, 5 op/s 2024-09-14T03:51:16.866 INFO:tasks.rados.rados.0.smithi110.stdout:3945: done (1 left) 2024-09-14T03:51:16.867 INFO:tasks.rados.rados.0.smithi110.stdout:3946: copy_from oid 18 from oid 46 current snap is 392 2024-09-14T03:51:16.867 INFO:tasks.rados.rados.0.smithi110.stdout:3947: snap_create 2024-09-14T03:51:16.867 INFO:tasks.rados.rados.0.smithi110.stdout:3944: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:51:16.867 INFO:tasks.rados.rados.0.smithi110.stdout:3944: finishing copy_from to smithi11075562-15 2024-09-14T03:51:16.867 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2014 (ObjNum 1163 snap 379 seq_num 1163) dirty exists 2024-09-14T03:51:16.902 INFO:tasks.rados.rados.0.smithi110.stdout:3946: finishing copy_from racing read to smithi11075562-18 2024-09-14T03:51:16.904 INFO:tasks.rados.rados.0.smithi110.stdout:3946: finishing copy_from to smithi11075562-18 2024-09-14T03:51:16.904 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 18 v 2047 (ObjNum 1067 snap 336 seq_num 1067) dirty exists 2024-09-14T03:51:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:16 smithi110 ceph-mon[88262]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T03:51:17.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:16 smithi110 ceph-mon[88262]: pgmap v927: 121 pgs: 121 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 MiB/s rd, 600 KiB/s wr, 9 op/s 2024-09-14T03:51:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:16 smithi110 ceph-mon[86423]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T03:51:17.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:16 smithi110 ceph-mon[86423]: pgmap v927: 121 pgs: 121 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 MiB/s rd, 600 KiB/s wr, 9 op/s 2024-09-14T03:51:17.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:16 smithi195 ceph-mon[82388]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T03:51:17.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:16 smithi195 ceph-mon[82388]: pgmap v927: 121 pgs: 121 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 MiB/s rd, 600 KiB/s wr, 9 op/s 2024-09-14T03:51:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:3944: done (2 left) 2024-09-14T03:51:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:3946: done (1 left) 2024-09-14T03:51:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:3947: done (0 left) 2024-09-14T03:51:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:3948: read oid 18 snap -1 2024-09-14T03:51:17.867 INFO:tasks.rados.rados.0.smithi110.stdout:3948: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:51:17.868 INFO:tasks.rados.rados.0.smithi110.stdout:3949: read oid 24 snap 370 2024-09-14T03:51:17.868 INFO:tasks.rados.rados.0.smithi110.stdout:3949: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:51:17.868 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 21 current snap is 393 2024-09-14T03:51:17.868 INFO:tasks.rados.rados.0.smithi110.stdout:3950: seq_num 1202 ranges {0=540672,540672=401408} 2024-09-14T03:51:17.870 INFO:tasks.rados.rados.0.smithi110.stdout:3950: writing smithi11075562-21 from 0 to 540672 tid 1 2024-09-14T03:51:17.873 INFO:tasks.rados.rados.0.smithi110.stdout:3950: writing smithi11075562-21 from 540672 to 942080 tid 2 2024-09-14T03:51:17.873 INFO:tasks.rados.rados.0.smithi110.stdout:3951: copy_from oid 15 from oid 29 current snap is 393 2024-09-14T03:51:17.873 INFO:tasks.rados.rados.0.smithi110.stdout:3952: read oid 40 snap -1 2024-09-14T03:51:17.873 INFO:tasks.rados.rados.0.smithi110.stdout:3952: expect deleted 2024-09-14T03:51:17.874 INFO:tasks.rados.rados.0.smithi110.stdout:3953: copy_from oid 7 from oid 19 current snap is 393 2024-09-14T03:51:17.874 INFO:tasks.rados.rados.0.smithi110.stdout:3954: rollback oid 34 current snap is 393 2024-09-14T03:51:17.874 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 34 to 387 2024-09-14T03:51:17.874 INFO:tasks.rados.rados.0.smithi110.stdout:3955: rollback oid 3 current snap is 393 2024-09-14T03:51:17.874 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 3 to 371 2024-09-14T03:51:17.874 INFO:tasks.rados.rados.0.smithi110.stdout:3956: snap_remove snap 391 2024-09-14T03:51:18.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:17 smithi110 ceph-mon[88262]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T03:51:18.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:17 smithi110 ceph-mon[86423]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T03:51:18.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:17 smithi195 ceph-mon[82388]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T03:51:18.847 INFO:tasks.rados.rados.0.smithi110.stdout:3956: done (8 left) 2024-09-14T03:51:18.847 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 27 current snap is 393 2024-09-14T03:51:18.848 INFO:tasks.rados.rados.0.smithi110.stdout:3957: seq_num 1203 ranges {786432=557056,1343488=401408} 2024-09-14T03:51:18.855 INFO:tasks.rados.rados.0.smithi110.stdout:3957: writing smithi11075562-27 from 786432 to 1343488 tid 1 2024-09-14T03:51:18.859 INFO:tasks.rados.rados.0.smithi110.stdout:3957: writing smithi11075562-27 from 1343488 to 1744896 tid 2 2024-09-14T03:51:18.859 INFO:tasks.rados.rados.0.smithi110.stdout:3958: read oid 28 snap -1 2024-09-14T03:51:18.859 INFO:tasks.rados.rados.0.smithi110.stdout:3958: expect (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:51:18.860 INFO:tasks.rados.rados.0.smithi110.stdout:3959: copy_from oid 46 from oid 17 current snap is 393 2024-09-14T03:51:18.860 INFO:tasks.rados.rados.0.smithi110.stdout:3950: finishing write tid 1 to smithi11075562-21 2024-09-14T03:51:18.860 INFO:tasks.rados.rados.0.smithi110.stdout:3950: finishing write tid 2 to smithi11075562-21 2024-09-14T03:51:18.860 INFO:tasks.rados.rados.0.smithi110.stdout:3950: finishing write tid 3 to smithi11075562-21 2024-09-14T03:51:18.860 INFO:tasks.rados.rados.0.smithi110.stdout:3954: finishing rollback tid 0 to smithi11075562-34 2024-09-14T03:51:18.860 INFO:tasks.rados.rados.0.smithi110.stdout:3954: finishing rollback tid 1 to smithi11075562-34 2024-09-14T03:51:18.860 INFO:tasks.rados.rados.0.smithi110.stdout:3955: finishing rollback tid 0 to smithi11075562-3 2024-09-14T03:51:18.860 INFO:tasks.rados.rados.0.smithi110.stdout:3955: finishing rollback tid 1 to smithi11075562-3 2024-09-14T03:51:18.861 INFO:tasks.rados.rados.0.smithi110.stdout:3950: finishing write tid 4 to smithi11075562-21 2024-09-14T03:51:18.861 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 21 v 2525 (ObjNum 1202 snap 393 seq_num 1202) dirty exists 2024-09-14T03:51:18.862 INFO:tasks.rados.rados.0.smithi110.stdout:3950: left oid 21 (ObjNum 1202 snap 393 seq_num 1202) 2024-09-14T03:51:18.862 INFO:tasks.rados.rados.0.smithi110.stdout:3955: finishing rollback tid 2 to smithi11075562-3 2024-09-14T03:51:18.862 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 2529 (ObjNum 1124 snap 366 seq_num 1124) dirty exists 2024-09-14T03:51:18.862 INFO:tasks.rados.rados.0.smithi110.stdout:3953: finishing copy_from racing read to smithi11075562-7 2024-09-14T03:51:18.862 INFO:tasks.rados.rados.0.smithi110.stdout:3954: finishing rollback tid 2 to smithi11075562-34 2024-09-14T03:51:18.862 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 34 v 2184 (ObjNum 129 snap 0 seq_num 66514840) dirty exists 2024-09-14T03:51:18.863 INFO:tasks.rados.rados.0.smithi110.stdout:3953: finishing copy_from to smithi11075562-7 2024-09-14T03:51:18.863 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 2526 (ObjNum 1164 snap 379 seq_num 1164) dirty exists 2024-09-14T03:51:18.863 INFO:tasks.rados.rados.0.smithi110.stdout:3951: finishing copy_from racing read to smithi11075562-15 2024-09-14T03:51:18.863 INFO:tasks.rados.rados.0.smithi110.stdout:3949: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:51:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:18 smithi110 ceph-mon[88262]: pgmap v929: 121 pgs: 121 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 320 KiB/s wr, 7 op/s 2024-09-14T03:51:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:18 smithi110 ceph-mon[88262]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T03:51:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:18 smithi110 ceph-mon[88262]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T03:51:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:18 smithi110 ceph-mon[86423]: pgmap v929: 121 pgs: 121 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 320 KiB/s wr, 7 op/s 2024-09-14T03:51:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:18 smithi110 ceph-mon[86423]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T03:51:19.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:18 smithi110 ceph-mon[86423]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T03:51:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:18 smithi195 ceph-mon[82388]: pgmap v929: 121 pgs: 121 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 320 KiB/s wr, 7 op/s 2024-09-14T03:51:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:18 smithi195 ceph-mon[82388]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T03:51:19.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:18 smithi195 ceph-mon[82388]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T03:51:19.573 INFO:tasks.rados.rados.0.smithi110.stdout:3951: finishing copy_from to smithi11075562-15 2024-09-14T03:51:19.573 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 1659 (ObjNum 1188 snap 389 seq_num 1188) dirty exists 2024-09-14T03:51:19.573 INFO:tasks.rados.rados.0.smithi110.stdout:3948: expect (ObjNum 1067 snap 336 seq_num 1067) 2024-09-14T03:51:20.012 INFO:tasks.rados.rados.0.smithi110.stdout:3959: finishing copy_from racing read to smithi11075562-46 2024-09-14T03:51:20.012 INFO:tasks.rados.rados.0.smithi110.stdout:3957: finishing write tid 1 to smithi11075562-27 2024-09-14T03:51:20.012 INFO:tasks.rados.rados.0.smithi110.stdout:3959: finishing copy_from to smithi11075562-46 2024-09-14T03:51:20.012 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 46 v 2014 (ObjNum 1163 snap 379 seq_num 1163) dirty exists 2024-09-14T03:51:20.012 INFO:tasks.rados.rados.0.smithi110.stdout:3957: finishing write tid 2 to smithi11075562-27 2024-09-14T03:51:20.013 INFO:tasks.rados.rados.0.smithi110.stdout:3957: finishing write tid 3 to smithi11075562-27 2024-09-14T03:51:20.013 INFO:tasks.rados.rados.0.smithi110.stdout:3957: finishing write tid 4 to smithi11075562-27 2024-09-14T03:51:20.013 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 27 v 1823 (ObjNum 1203 snap 393 seq_num 1203) dirty exists 2024-09-14T03:51:20.013 INFO:tasks.rados.rados.0.smithi110.stdout:3957: left oid 27 (ObjNum 1203 snap 393 seq_num 1203) 2024-09-14T03:51:20.013 INFO:tasks.rados.rados.0.smithi110.stdout:3958: expect (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:51:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:51:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:51:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:21.156 INFO:tasks.rados.rados.0.smithi110.stdout:3948: done (10 left) 2024-09-14T03:51:21.156 INFO:tasks.rados.rados.0.smithi110.stdout:3949: done (9 left) 2024-09-14T03:51:21.156 INFO:tasks.rados.rados.0.smithi110.stdout:3950: done (8 left) 2024-09-14T03:51:21.156 INFO:tasks.rados.rados.0.smithi110.stdout:3951: done (7 left) 2024-09-14T03:51:21.156 INFO:tasks.rados.rados.0.smithi110.stdout:3952: done (6 left) 2024-09-14T03:51:21.157 INFO:tasks.rados.rados.0.smithi110.stdout:3953: done (5 left) 2024-09-14T03:51:21.157 INFO:tasks.rados.rados.0.smithi110.stdout:3954: done (4 left) 2024-09-14T03:51:21.157 INFO:tasks.rados.rados.0.smithi110.stdout:3955: done (3 left) 2024-09-14T03:51:21.157 INFO:tasks.rados.rados.0.smithi110.stdout:3957: done (2 left) 2024-09-14T03:51:21.157 INFO:tasks.rados.rados.0.smithi110.stdout:3958: done (1 left) 2024-09-14T03:51:21.157 INFO:tasks.rados.rados.0.smithi110.stdout:3959: done (0 left) 2024-09-14T03:51:21.157 INFO:tasks.rados.rados.0.smithi110.stdout:3960: delete oid 6 current snap is 393 2024-09-14T03:51:21.160 INFO:tasks.rados.rados.0.smithi110.stdout:3960: done (0 left) 2024-09-14T03:51:21.160 INFO:tasks.rados.rados.0.smithi110.stdout:3961: snap_remove snap 370 2024-09-14T03:51:21.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:20 smithi110 ceph-mon[88262]: pgmap v932: 121 pgs: 121 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-14T03:51:21.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:20 smithi110 ceph-mon[86423]: pgmap v932: 121 pgs: 121 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-14T03:51:21.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:20 smithi195 ceph-mon[82388]: pgmap v932: 121 pgs: 121 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-14T03:51:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3961: done (0 left) 2024-09-14T03:51:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3962: read oid 24 snap -1 2024-09-14T03:51:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3962: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:51:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 1 current snap is 393 2024-09-14T03:51:21.886 INFO:tasks.rados.rados.0.smithi110.stdout:3963: seq_num 1205 ranges {0=589824,589824=401408} 2024-09-14T03:51:21.889 INFO:tasks.rados.rados.0.smithi110.stdout:3963: writing smithi11075562-1 from 0 to 589824 tid 1 2024-09-14T03:51:21.892 INFO:tasks.rados.rados.0.smithi110.stdout:3963: writing smithi11075562-1 from 589824 to 991232 tid 2 2024-09-14T03:51:21.892 INFO:tasks.rados.rados.0.smithi110.stdout:3964: snap_remove snap 392 2024-09-14T03:51:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:51:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:51:21] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.51.0" 2024-09-14T03:51:22.891 INFO:tasks.rados.rados.0.smithi110.stdout:3963: finishing write tid 1 to smithi11075562-1 2024-09-14T03:51:22.891 INFO:tasks.rados.rados.0.smithi110.stdout:3963: finishing write tid 2 to smithi11075562-1 2024-09-14T03:51:22.891 INFO:tasks.rados.rados.0.smithi110.stdout:3963: finishing write tid 3 to smithi11075562-1 2024-09-14T03:51:22.891 INFO:tasks.rados.rados.0.smithi110.stdout:3963: finishing write tid 4 to smithi11075562-1 2024-09-14T03:51:22.891 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 1 v 2516 (ObjNum 1205 snap 393 seq_num 1205) dirty exists 2024-09-14T03:51:22.891 INFO:tasks.rados.rados.0.smithi110.stdout:3963: left oid 1 (ObjNum 1205 snap 393 seq_num 1205) 2024-09-14T03:51:22.891 INFO:tasks.rados.rados.0.smithi110.stdout:3962: expect (ObjNum 1082 snap 342 seq_num 1082) 2024-09-14T03:51:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:22 smithi110 ceph-mon[86423]: pgmap v933: 121 pgs: 121 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 8 op/s 2024-09-14T03:51:23.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:22 smithi110 ceph-mon[86423]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T03:51:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:22 smithi110 ceph-mon[88262]: pgmap v933: 121 pgs: 121 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 8 op/s 2024-09-14T03:51:23.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:22 smithi110 ceph-mon[88262]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T03:51:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:22 smithi195 ceph-mon[82388]: pgmap v933: 121 pgs: 121 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 8 op/s 2024-09-14T03:51:23.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:22 smithi195 ceph-mon[82388]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T03:51:23.606 INFO:tasks.rados.rados.0.smithi110.stdout:3962: done (2 left) 2024-09-14T03:51:23.606 INFO:tasks.rados.rados.0.smithi110.stdout:3963: done (1 left) 2024-09-14T03:51:23.606 INFO:tasks.rados.rados.0.smithi110.stdout:3964: done (0 left) 2024-09-14T03:51:23.607 INFO:tasks.rados.rados.0.smithi110.stdout:3965: read oid 12 snap -1 2024-09-14T03:51:23.607 INFO:tasks.rados.rados.0.smithi110.stdout:3965: expect (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:51:23.607 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 3 current snap is 393 2024-09-14T03:51:23.607 INFO:tasks.rados.rados.0.smithi110.stdout:3966: seq_num 1206 ranges {5308416=606208,5914624=401408} 2024-09-14T03:51:23.637 INFO:tasks.rados.rados.0.smithi110.stdout:3966: writing smithi11075562-3 from 5308416 to 5914624 tid 1 2024-09-14T03:51:23.640 INFO:tasks.rados.rados.0.smithi110.stdout:3966: writing smithi11075562-3 from 5914624 to 6316032 tid 2 2024-09-14T03:51:23.640 INFO:tasks.rados.rados.0.smithi110.stdout:3967: copy_from oid 44 from oid 4 current snap is 393 2024-09-14T03:51:23.640 INFO:tasks.rados.rados.0.smithi110.stdout:3968: rmattr oid 48 current snap is 393 2024-09-14T03:51:23.640 INFO:tasks.rados.rados.0.smithi110.stdout:3968: done (3 left) 2024-09-14T03:51:23.640 INFO:tasks.rados.rados.0.smithi110.stdout:3969: read oid 47 snap -1 2024-09-14T03:51:23.640 INFO:tasks.rados.rados.0.smithi110.stdout:3969: expect (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:51:23.640 INFO:tasks.rados.rados.0.smithi110.stdout:3970: snap_create 2024-09-14T03:51:23.643 INFO:tasks.rados.rados.0.smithi110.stdout:3966: finishing write tid 1 to smithi11075562-3 2024-09-14T03:51:23.643 INFO:tasks.rados.rados.0.smithi110.stdout:3966: finishing write tid 2 to smithi11075562-3 2024-09-14T03:51:23.643 INFO:tasks.rados.rados.0.smithi110.stdout:3966: finishing write tid 3 to smithi11075562-3 2024-09-14T03:51:23.645 INFO:tasks.rados.rados.0.smithi110.stdout:3966: finishing write tid 4 to smithi11075562-3 2024-09-14T03:51:23.645 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 3 v 2532 (ObjNum 1206 snap 393 seq_num 1206) dirty exists 2024-09-14T03:51:23.645 INFO:tasks.rados.rados.0.smithi110.stdout:3966: left oid 3 (ObjNum 1206 snap 393 seq_num 1206) 2024-09-14T03:51:23.676 INFO:tasks.rados.rados.0.smithi110.stdout:3965: expect (ObjNum 1113 snap 354 seq_num 1113) 2024-09-14T03:51:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:23 smithi110 ceph-mon[88262]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T03:51:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:23 smithi110 ceph-mon[88262]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T03:51:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:23 smithi110 ceph-mon[86423]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T03:51:24.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:23 smithi110 ceph-mon[86423]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T03:51:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:23 smithi195 ceph-mon[82388]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T03:51:24.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:23 smithi195 ceph-mon[82388]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T03:51:24.829 INFO:tasks.rados.rados.0.smithi110.stdout:3969: expect (ObjNum 1092 snap 343 seq_num 1092) 2024-09-14T03:51:24.870 INFO:tasks.rados.rados.0.smithi110.stdout:3967: finishing copy_from racing read to smithi11075562-44 2024-09-14T03:51:24.870 INFO:tasks.rados.rados.0.smithi110.stdout:3967: finishing copy_from to smithi11075562-44 2024-09-14T03:51:24.870 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 2179 (ObjNum 1172 snap 382 seq_num 1172) dirty exists 2024-09-14T03:51:24.870 INFO:tasks.rados.rados.0.smithi110.stdout:3965: done (4 left) 2024-09-14T03:51:24.870 INFO:tasks.rados.rados.0.smithi110.stdout:3966: done (3 left) 2024-09-14T03:51:24.870 INFO:tasks.rados.rados.0.smithi110.stdout:3967: done (2 left) 2024-09-14T03:51:24.870 INFO:tasks.rados.rados.0.smithi110.stdout:3969: done (1 left) 2024-09-14T03:51:24.871 INFO:tasks.rados.rados.0.smithi110.stdout:3970: done (0 left) 2024-09-14T03:51:24.871 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 42 current snap is 394 2024-09-14T03:51:24.871 INFO:tasks.rados.rados.0.smithi110.stdout:3971: seq_num 1207 ranges {6905856=622592,7528448=401408} 2024-09-14T03:51:24.909 INFO:tasks.rados.rados.0.smithi110.stdout:3971: writing smithi11075562-42 from 6905856 to 7528448 tid 1 2024-09-14T03:51:24.911 INFO:tasks.rados.rados.0.smithi110.stdout:3971: writing smithi11075562-42 from 7528448 to 7929856 tid 2 2024-09-14T03:51:24.911 INFO:tasks.rados.rados.0.smithi110.stdout:3972: read oid 41 snap -1 2024-09-14T03:51:24.911 INFO:tasks.rados.rados.0.smithi110.stdout:3972: expect (ObjNum 1195 snap 391 seq_num 1195) 2024-09-14T03:51:24.911 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 394 2024-09-14T03:51:24.911 INFO:tasks.rados.rados.0.smithi110.stdout:3973: seq_num 1208 ranges {0=638976,638976=401408} 2024-09-14T03:51:24.914 INFO:tasks.rados.rados.0.smithi110.stdout:3973: writing smithi11075562-49 from 0 to 638976 tid 1 2024-09-14T03:51:24.917 INFO:tasks.rados.rados.0.smithi110.stdout:3973: writing smithi11075562-49 from 638976 to 1040384 tid 2 2024-09-14T03:51:24.917 INFO:tasks.rados.rados.0.smithi110.stdout:3974: delete oid 21 current snap is 394 2024-09-14T03:51:24.917 INFO:tasks.rados.rados.0.smithi110.stdout:3971: finishing write tid 1 to smithi11075562-42 2024-09-14T03:51:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3971: finishing write tid 2 to smithi11075562-42 2024-09-14T03:51:24.918 INFO:tasks.rados.rados.0.smithi110.stdout:3971: finishing write tid 3 to smithi11075562-42 2024-09-14T03:51:24.921 INFO:tasks.rados.rados.0.smithi110.stdout:3973: finishing write tid 1 to smithi11075562-49 2024-09-14T03:51:24.921 INFO:tasks.rados.rados.0.smithi110.stdout:3971: finishing write tid 4 to smithi11075562-42 2024-09-14T03:51:24.922 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 42 v 2519 (ObjNum 1207 snap 394 seq_num 1207) dirty exists 2024-09-14T03:51:24.922 INFO:tasks.rados.rados.0.smithi110.stdout:3971: left oid 42 (ObjNum 1207 snap 394 seq_num 1207) 2024-09-14T03:51:24.922 INFO:tasks.rados.rados.0.smithi110.stdout:3973: finishing write tid 2 to smithi11075562-49 2024-09-14T03:51:24.922 INFO:tasks.rados.rados.0.smithi110.stdout:3973: finishing write tid 3 to smithi11075562-49 2024-09-14T03:51:24.923 INFO:tasks.rados.rados.0.smithi110.stdout:3973: finishing write tid 4 to smithi11075562-49 2024-09-14T03:51:24.923 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2522 (ObjNum 1208 snap 394 seq_num 1208) dirty exists 2024-09-14T03:51:24.923 INFO:tasks.rados.rados.0.smithi110.stdout:3973: left oid 49 (ObjNum 1208 snap 394 seq_num 1208) 2024-09-14T03:51:24.931 INFO:tasks.rados.rados.0.smithi110.stdout:3971: done (3 left) 2024-09-14T03:51:24.931 INFO:tasks.rados.rados.0.smithi110.stdout:3973: done (2 left) 2024-09-14T03:51:24.931 INFO:tasks.rados.rados.0.smithi110.stdout:3974: done (1 left) 2024-09-14T03:51:24.931 INFO:tasks.rados.rados.0.smithi110.stdout:3975: snap_remove snap 374 2024-09-14T03:51:25.854 INFO:tasks.rados.rados.0.smithi110.stdout:3975: done (1 left) 2024-09-14T03:51:25.854 INFO:tasks.rados.rados.0.smithi110.stdout:3976: rmattr oid 44 current snap is 394 2024-09-14T03:51:25.854 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 15 current snap is 394 2024-09-14T03:51:25.854 INFO:tasks.rados.rados.0.smithi110.stdout:3977: seq_num 1210 ranges {4636672=671744,5308416=401408} 2024-09-14T03:51:25.884 INFO:tasks.rados.rados.0.smithi110.stdout:3977: writing smithi11075562-15 from 4636672 to 5308416 tid 1 2024-09-14T03:51:25.887 INFO:tasks.rados.rados.0.smithi110.stdout:3977: writing smithi11075562-15 from 5308416 to 5709824 tid 2 2024-09-14T03:51:25.887 INFO:tasks.rados.rados.0.smithi110.stdout:3978: rollback oid 49 current snap is 394 2024-09-14T03:51:25.887 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 49 to 363 2024-09-14T03:51:25.887 INFO:tasks.rados.rados.0.smithi110.stdout:3972: expect (ObjNum 1195 snap 391 seq_num 1195) 2024-09-14T03:51:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:25 smithi110 ceph-mon[88262]: pgmap v937: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 11 op/s 2024-09-14T03:51:26.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:25 smithi110 ceph-mon[86423]: pgmap v937: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 11 op/s 2024-09-14T03:51:26.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:25 smithi195 ceph-mon[82388]: pgmap v937: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 11 op/s 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 44 v 2520 (ObjNum 1172 snap 382 seq_num 1172) dirty exists 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:3978: finishing rollback tid 0 to smithi11075562-49 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:3978: finishing rollback tid 1 to smithi11075562-49 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2524 (ObjNum 5633 snap 0 seq_num 2917114672) dirty dne 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:3977: finishing write tid 1 to smithi11075562-15 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:3977: finishing write tid 2 to smithi11075562-15 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:3977: finishing write tid 3 to smithi11075562-15 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:3977: finishing write tid 4 to smithi11075562-15 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 15 v 2540 (ObjNum 1210 snap 394 seq_num 1210) dirty exists 2024-09-14T03:51:26.781 INFO:tasks.rados.rados.0.smithi110.stdout:3977: left oid 15 (ObjNum 1210 snap 394 seq_num 1210) 2024-09-14T03:51:26.782 INFO:tasks.rados.rados.0.smithi110.stdout:3972: done (3 left) 2024-09-14T03:51:26.782 INFO:tasks.rados.rados.0.smithi110.stdout:3976: done (2 left) 2024-09-14T03:51:26.782 INFO:tasks.rados.rados.0.smithi110.stdout:3977: done (1 left) 2024-09-14T03:51:26.782 INFO:tasks.rados.rados.0.smithi110.stdout:3978: done (0 left) 2024-09-14T03:51:26.782 INFO:tasks.rados.rados.0.smithi110.stdout:3979: copy_from oid 31 from oid 25 current snap is 394 2024-09-14T03:51:26.782 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 49 current snap is 394 2024-09-14T03:51:26.782 INFO:tasks.rados.rados.0.smithi110.stdout:3980: seq_num 1211 ranges {0=688128,688128=401408} 2024-09-14T03:51:26.782 INFO:tasks.rados.rados.0.smithi110.stdout:3980: writing smithi11075562-49 from 0 to 688128 tid 1 2024-09-14T03:51:26.784 INFO:tasks.rados.rados.0.smithi110.stdout:3980: writing smithi11075562-49 from 688128 to 1089536 tid 2 2024-09-14T03:51:26.784 INFO:tasks.rados.rados.0.smithi110.stdout:3981: delete oid 9 current snap is 394 2024-09-14T03:51:26.784 INFO:tasks.rados.rados.0.smithi110.stdout:3979: finishing copy_from racing read to smithi11075562-31 2024-09-14T03:51:26.786 INFO:tasks.rados.rados.0.smithi110.stdout:3979: finishing copy_from to smithi11075562-31 2024-09-14T03:51:26.786 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 31 v 2735 (ObjNum 1182 snap 388 seq_num 1182) dirty exists 2024-09-14T03:51:26.787 INFO:tasks.rados.rados.0.smithi110.stdout:3979: done (2 left) 2024-09-14T03:51:26.787 INFO:tasks.rados.rados.0.smithi110.stdout:3981: done (1 left) 2024-09-14T03:51:26.787 INFO:tasks.rados.rados.0.smithi110.stdout:3982: copy_from oid 9 from oid 36 current snap is 394 2024-09-14T03:51:26.787 INFO:tasks.rados.rados.0.smithi110.stdout:3983: snap_create 2024-09-14T03:51:26.789 INFO:tasks.rados.rados.0.smithi110.stdout:3980: finishing write tid 1 to smithi11075562-49 2024-09-14T03:51:26.789 INFO:tasks.rados.rados.0.smithi110.stdout:3980: finishing write tid 2 to smithi11075562-49 2024-09-14T03:51:26.790 INFO:tasks.rados.rados.0.smithi110.stdout:3980: finishing write tid 3 to smithi11075562-49 2024-09-14T03:51:26.791 INFO:tasks.rados.rados.0.smithi110.stdout:3980: finishing write tid 4 to smithi11075562-49 2024-09-14T03:51:26.792 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 49 v 2527 (ObjNum 1211 snap 394 seq_num 1211) dirty exists 2024-09-14T03:51:26.792 INFO:tasks.rados.rados.0.smithi110.stdout:3980: left oid 49 (ObjNum 1211 snap 394 seq_num 1211) 2024-09-14T03:51:26.794 INFO:tasks.rados.rados.0.smithi110.stdout:3982: finishing copy_from racing read to smithi11075562-9 2024-09-14T03:51:26.795 INFO:tasks.rados.rados.0.smithi110.stdout:3982: finishing copy_from to smithi11075562-9 2024-09-14T03:51:26.795 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 9 v 2267 (ObjNum 1157 snap 378 seq_num 1157) dirty exists 2024-09-14T03:51:26.858 INFO:tasks.rados.rados.0.smithi110.stdout:3980: done (2 left) 2024-09-14T03:51:26.858 INFO:tasks.rados.rados.0.smithi110.stdout:3982: done (1 left) 2024-09-14T03:51:26.858 INFO:tasks.rados.rados.0.smithi110.stdout:3983: done (0 left) 2024-09-14T03:51:26.858 INFO:tasks.rados.rados.0.smithi110.stdout:3984: rollback oid 35 current snap is 395 2024-09-14T03:51:26.858 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 387 2024-09-14T03:51:26.858 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 45 current snap is 395 2024-09-14T03:51:26.858 INFO:tasks.rados.rados.0.smithi110.stdout:3985: seq_num 1213 ranges {638976=720896,1359872=401408} 2024-09-14T03:51:26.865 INFO:tasks.rados.rados.0.smithi110.stdout:3985: writing smithi11075562-45 from 638976 to 1359872 tid 1 2024-09-14T03:51:26.868 INFO:tasks.rados.rados.0.smithi110.stdout:3985: writing smithi11075562-45 from 1359872 to 1761280 tid 2 2024-09-14T03:51:26.868 INFO:tasks.rados.rados.0.smithi110.stdout:3986: snap_remove snap 394 2024-09-14T03:51:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:26 smithi110 ceph-mon[86423]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T03:51:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:26 smithi110 ceph-mon[86423]: pgmap v939: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 620 KiB/s wr, 5 op/s 2024-09-14T03:51:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:26 smithi110 ceph-mon[88262]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T03:51:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:26 smithi110 ceph-mon[88262]: pgmap v939: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 620 KiB/s wr, 5 op/s 2024-09-14T03:51:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:26 smithi195 ceph-mon[82388]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T03:51:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:26 smithi195 ceph-mon[82388]: pgmap v939: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 620 KiB/s wr, 5 op/s 2024-09-14T03:51:27.863 INFO:tasks.rados.rados.0.smithi110.stdout:3986: done (2 left) 2024-09-14T03:51:27.863 INFO:tasks.rados.rados.0.smithi110.stdout:3987: snap_create 2024-09-14T03:51:27.863 INFO:tasks.rados.rados.0.smithi110.stdout:3984: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:51:27.863 INFO:tasks.rados.rados.0.smithi110.stdout:3984: finishing rollback tid 2 to smithi11075562-35 2024-09-14T03:51:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 2529 (ObjNum 1157 snap 378 seq_num 1157) dirty exists 2024-09-14T03:51:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:3985: finishing write tid 1 to smithi11075562-45 2024-09-14T03:51:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:3985: finishing write tid 2 to smithi11075562-45 2024-09-14T03:51:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:3985: finishing write tid 3 to smithi11075562-45 2024-09-14T03:51:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:3985: finishing write tid 4 to smithi11075562-45 2024-09-14T03:51:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 45 v 2854 (ObjNum 1213 snap 395 seq_num 1213) dirty exists 2024-09-14T03:51:27.864 INFO:tasks.rados.rados.0.smithi110.stdout:3985: left oid 45 (ObjNum 1213 snap 395 seq_num 1213) 2024-09-14T03:51:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:27 smithi110 ceph-mon[88262]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T03:51:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:27 smithi110 ceph-mon[86423]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T03:51:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:27 smithi195 ceph-mon[82388]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T03:51:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:28.849 INFO:tasks.rados.rados.0.smithi110.stdout:3984: done (2 left) 2024-09-14T03:51:28.849 INFO:tasks.rados.rados.0.smithi110.stdout:3985: done (1 left) 2024-09-14T03:51:28.849 INFO:tasks.rados.rados.0.smithi110.stdout:3987: done (0 left) 2024-09-14T03:51:28.850 INFO:tasks.rados.rados.0.smithi110.stdout:3988: snap_remove snap 393 2024-09-14T03:51:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:28 smithi110 ceph-mon[88262]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T03:51:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:28 smithi110 ceph-mon[88262]: pgmap v942: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:51:29.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:28 smithi110 ceph-mon[88262]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T03:51:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:28 smithi110 ceph-mon[86423]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T03:51:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:28 smithi110 ceph-mon[86423]: pgmap v942: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:51:29.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:28 smithi110 ceph-mon[86423]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T03:51:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:28 smithi195 ceph-mon[82388]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T03:51:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:28 smithi195 ceph-mon[82388]: pgmap v942: 121 pgs: 121 active+clean; 362 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:51:29.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:28 smithi195 ceph-mon[82388]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T03:51:29.852 INFO:tasks.rados.rados.0.smithi110.stdout:3988: done (0 left) 2024-09-14T03:51:29.852 INFO:tasks.rados.rados.0.smithi110.stdout:3989: rollback oid 37 current snap is 396 2024-09-14T03:51:29.852 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 37 to 395 2024-09-14T03:51:29.853 INFO:tasks.rados.rados.0.smithi110.stdout:append oid 36 current snap is 396 2024-09-14T03:51:29.853 INFO:tasks.rados.rados.0.smithi110.stdout:3990: seq_num 1214 ranges {1138688=737280,1875968=401408} 2024-09-14T03:51:29.861 INFO:tasks.rados.rados.0.smithi110.stdout:3990: writing smithi11075562-36 from 1138688 to 1875968 tid 1 2024-09-14T03:51:29.865 INFO:tasks.rados.rados.0.smithi110.stdout:3990: writing smithi11075562-36 from 1875968 to 2277376 tid 2 2024-09-14T03:51:29.865 INFO:tasks.rados.rados.0.smithi110.stdout:3991: read oid 21 snap -1 2024-09-14T03:51:29.865 INFO:tasks.rados.rados.0.smithi110.stdout:3991: expect deleted 2024-09-14T03:51:29.866 INFO:tasks.rados.rados.0.smithi110.stdout:3992: rollback oid 30 current snap is 396 2024-09-14T03:51:29.866 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 30 to 371 2024-09-14T03:51:29.866 INFO:tasks.rados.rados.0.smithi110.stdout:3993: rmattr oid 43 current snap is 396 2024-09-14T03:51:29.866 INFO:tasks.rados.rados.0.smithi110.stdout:3994: rollback oid 35 current snap is 396 2024-09-14T03:51:29.866 INFO:tasks.rados.rados.0.smithi110.stdout:rollback oid 35 to 387 2024-09-14T03:51:29.866 INFO:tasks.rados.rados.0.smithi110.stdout:3995: snap_create 2024-09-14T03:51:29.868 INFO:tasks.rados.rados.0.smithi110.stdout:3989: finishing rollback tid 0 to smithi11075562-37 2024-09-14T03:51:29.869 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 43 v 2527 (ObjNum 1191 snap 390 seq_num 1191) dirty exists 2024-09-14T03:51:29.870 INFO:tasks.rados.rados.0.smithi110.stdout:3994: finishing rollback tid 0 to smithi11075562-35 2024-09-14T03:51:29.873 INFO:tasks.rados.rados.0.smithi110.stdout:3994: finishing rollback tid 1 to smithi11075562-35 2024-09-14T03:51:29.873 INFO:tasks.rados.rados.0.smithi110.stdout:3992: finishing rollback tid 0 to smithi11075562-30 2024-09-14T03:51:29.873 INFO:tasks.rados.rados.0.smithi110.stdout:3994: finishing rollback tid 2 to smithi11075562-35 2024-09-14T03:51:29.874 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 35 v 2532 (ObjNum 1157 snap 378 seq_num 1157) dirty exists 2024-09-14T03:51:29.874 INFO:tasks.rados.rados.0.smithi110.stdout:3989: finishing rollback tid 1 to smithi11075562-37 2024-09-14T03:51:29.874 INFO:tasks.rados.rados.0.smithi110.stdout:3989: finishing rollback tid 2 to smithi11075562-37 2024-09-14T03:51:29.874 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 37 v 2860 (ObjNum 1082 snap 342 seq_num 1082) dirty exists 2024-09-14T03:51:29.874 INFO:tasks.rados.rados.0.smithi110.stdout:3990: finishing write tid 1 to smithi11075562-36 2024-09-14T03:51:29.874 INFO:tasks.rados.rados.0.smithi110.stdout:3992: finishing rollback tid 1 to smithi11075562-30 2024-09-14T03:51:29.875 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 30 v 2542 (ObjNum 737 snap 0 seq_num 2917113776) dirty dne 2024-09-14T03:51:29.876 INFO:tasks.rados.rados.0.smithi110.stdout:3990: finishing write tid 2 to smithi11075562-36 2024-09-14T03:51:29.877 INFO:tasks.rados.rados.0.smithi110.stdout:3990: finishing write tid 3 to smithi11075562-36 2024-09-14T03:51:29.879 INFO:tasks.rados.rados.0.smithi110.stdout:3990: finishing write tid 4 to smithi11075562-36 2024-09-14T03:51:29.879 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 36 v 2866 (ObjNum 1214 snap 396 seq_num 1214) dirty exists 2024-09-14T03:51:29.879 INFO:tasks.rados.rados.0.smithi110.stdout:3990: left oid 36 (ObjNum 1214 snap 396 seq_num 1214) 2024-09-14T03:51:30.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:51:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:51:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:30.856 INFO:tasks.rados.rados.0.smithi110.stdout:3989: done (6 left) 2024-09-14T03:51:30.856 INFO:tasks.rados.rados.0.smithi110.stdout:3990: done (5 left) 2024-09-14T03:51:30.856 INFO:tasks.rados.rados.0.smithi110.stdout:3991: done (4 left) 2024-09-14T03:51:30.856 INFO:tasks.rados.rados.0.smithi110.stdout:3992: done (3 left) 2024-09-14T03:51:30.856 INFO:tasks.rados.rados.0.smithi110.stdout:3993: done (2 left) 2024-09-14T03:51:30.856 INFO:tasks.rados.rados.0.smithi110.stdout:3994: done (1 left) 2024-09-14T03:51:30.856 INFO:tasks.rados.rados.0.smithi110.stdout:3995: done (0 left) 2024-09-14T03:51:30.856 INFO:tasks.rados.rados.0.smithi110.stdout:3996: delete oid 27 current snap is 397 2024-09-14T03:51:30.860 INFO:tasks.rados.rados.0.smithi110.stdout:3996: done (0 left) 2024-09-14T03:51:30.860 INFO:tasks.rados.rados.0.smithi110.stdout:3997: copy_from oid 7 from oid 11 current snap is 397 2024-09-14T03:51:30.860 INFO:tasks.rados.rados.0.smithi110.stdout:3998: snap_create 2024-09-14T03:51:30.874 INFO:tasks.rados.rados.0.smithi110.stdout:3997: finishing copy_from racing read to smithi11075562-7 2024-09-14T03:51:30.875 INFO:tasks.rados.rados.0.smithi110.stdout:3997: finishing copy_from to smithi11075562-7 2024-09-14T03:51:30.875 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 7 v 2053 (ObjNum 1045 snap 330 seq_num 1045) dirty exists 2024-09-14T03:51:31.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:30 smithi110 ceph-mon[88262]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T03:51:31.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:30 smithi110 ceph-mon[88262]: pgmap v945: 121 pgs: 121 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 4.9 MiB/s wr, 10 op/s 2024-09-14T03:51:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:30 smithi110 ceph-mon[86423]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T03:51:31.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:30 smithi110 ceph-mon[86423]: pgmap v945: 121 pgs: 121 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 4.9 MiB/s wr, 10 op/s 2024-09-14T03:51:31.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:30 smithi195 ceph-mon[82388]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T03:51:31.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:30 smithi195 ceph-mon[82388]: pgmap v945: 121 pgs: 121 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 4.9 MiB/s wr, 10 op/s 2024-09-14T03:51:31.857 INFO:tasks.rados.rados.0.smithi110.stdout:3997: done (1 left) 2024-09-14T03:51:31.857 INFO:tasks.rados.rados.0.smithi110.stdout:3998: done (0 left) 2024-09-14T03:51:31.858 INFO:tasks.rados.rados.0.smithi110.stdout:3999: rmattr oid 22 current snap is 398 2024-09-14T03:51:31.858 INFO:tasks.rados.rados.0.smithi110.stdout: waiting on 1 2024-09-14T03:51:31.858 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:51:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:51:31] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-14T03:51:31.862 INFO:tasks.rados.rados.0.smithi110.stdout:update_object_version oid 22 v 2543 (ObjNum 1196 snap 391 seq_num 1196) dirty exists 2024-09-14T03:51:31.862 INFO:tasks.rados.rados.0.smithi110.stdout:3999: done (0 left) 2024-09-14T03:51:31.865 INFO:tasks.rados.rados.0.smithi110.stderr:0 errors. 2024-09-14T03:51:31.865 INFO:tasks.rados.rados.0.smithi110.stderr: 2024-09-14T03:51:31.868 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:51:32.115 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:32.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:31 smithi110 ceph-mon[88262]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T03:51:32.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:31 smithi110 ceph-mon[86423]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T03:51:32.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:31 smithi195 ceph-mon[82388]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T03:51:33.165 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:32 smithi110 ceph-mon[88262]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T03:51:33.165 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:32 smithi110 ceph-mon[88262]: pgmap v948: 121 pgs: 121 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 767 B/s rd, 4.5 MiB/s wr, 8 op/s 2024-09-14T03:51:33.165 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:32 smithi110 ceph-mon[86423]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T03:51:33.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:32 smithi110 ceph-mon[86423]: pgmap v948: 121 pgs: 121 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 767 B/s rd, 4.5 MiB/s wr, 8 op/s 2024-09-14T03:51:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:32 smithi195 ceph-mon[82388]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T03:51:33.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:32 smithi195 ceph-mon[82388]: pgmap v948: 121 pgs: 121 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 767 B/s rd, 4.5 MiB/s wr, 8 op/s 2024-09-14T03:51:33.816 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:51:33.816 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":874,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:51:31.850802+0000","last_up_change":"2024-09-14T03:45:53.777419+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:33:57.860054+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:34:01.436541+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:34:50.939554+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-09-14T03:37:35.262844+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"874","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":476,"snap_epoch":874,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":311,"up_from":324,"up_thru":603,"down_at":312,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":4018695393},{"type":"v1","addr":"172.21.15.110:6803","nonce":4018695393}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":4018695393},{"type":"v1","addr":"172.21.15.110:6805","nonce":4018695393}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":4018695393},{"type":"v1","addr":"172.21.15.110:6809","nonce":4018695393}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":4018695393},{"type":"v1","addr":"172.21.15.110:6807","nonce":4018695393}]},"public_addr":"172.21.15.110:6803/4018695393","cluster_addr":"172.21.15.110:6805/4018695393","heartbeat_back_addr":"172.21.15.110:6809/4018695393","heartbeat_front_addr":"172.21.15.110:6807/4018695393","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":344,"up_from":358,"up_thru":552,"down_at":345,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":1059959418},{"type":"v1","addr":"172.21.15.110:6811","nonce":1059959418}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":1059959418},{"type":"v1","addr":"172.21.15.110:6813","nonce":1059959418}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":1059959418},{"type":"v1","addr":"172.21.15.110:6817","nonce":1059959418}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":1059959418},{"type":"v1","addr":"172.21.15.110:6815","nonce":1059959418}]},"public_addr":"172.21.15.110:6811/1059959418","cluster_addr":"172.21.15.110:6813/1059959418","heartbeat_back_addr":"172.21.15.110:6817/1059959418","heartbeat_front_addr":"172.21.15.110:6815/1059959418","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":376,"up_from":389,"up_thru":603,"down_at":377,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":2442181625},{"type":"v1","addr":"172.21.15.110:6819","nonce":2442181625}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":2442181625},{"type":"v1","addr":"172.21.15.110:6821","nonce":2442181625}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2442181625},{"type":"v1","addr":"172.21.15.110:6825","nonce":2442181625}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":2442181625},{"type":"v1","addr":"172.21.15.110:6823","nonce":2442181625}]},"public_addr":"172.21.15.110:6819/2442181625","cluster_addr":"172.21.15.110:6821/2442181625","heartbeat_back_addr":"172.21.15.110:6825/2442181625","heartbeat_front_addr":"172.21.15.110:6823/2442181625","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":409,"up_from":420,"up_thru":603,"down_at":410,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":570098450},{"type":"v1","addr":"172.21.15.110:6827","nonce":570098450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":570098450},{"type":"v1","addr":"172.21.15.110:6829","nonce":570098450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":570098450},{"type":"v1","addr":"172.21.15.110:6833","nonce":570098450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":570098450},{"type":"v1","addr":"172.21.15.110:6831","nonce":570098450}]},"public_addr":"172.21.15.110:6827/570098450","cluster_addr":"172.21.15.110:6829/570098450","heartbeat_back_addr":"172.21.15.110:6833/570098450","heartbeat_front_addr":"172.21.15.110:6831/570098450","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":440,"up_from":454,"up_thru":552,"down_at":441,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":2754374122},{"type":"v1","addr":"172.21.15.195:6801","nonce":2754374122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":2754374122},{"type":"v1","addr":"172.21.15.195:6803","nonce":2754374122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":2754374122},{"type":"v1","addr":"172.21.15.195:6807","nonce":2754374122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":2754374122},{"type":"v1","addr":"172.21.15.195:6805","nonce":2754374122}]},"public_addr":"172.21.15.195:6801/2754374122","cluster_addr":"172.21.15.195:6803/2754374122","heartbeat_back_addr":"172.21.15.195:6807/2754374122","heartbeat_front_addr":"172.21.15.195:6805/2754374122","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":37,"last_clean_end":474,"up_from":486,"up_thru":552,"down_at":475,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3630816993},{"type":"v1","addr":"172.21.15.195:6809","nonce":3630816993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3630816993},{"type":"v1","addr":"172.21.15.195:6811","nonce":3630816993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3630816993},{"type":"v1","addr":"172.21.15.195:6815","nonce":3630816993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3630816993},{"type":"v1","addr":"172.21.15.195:6813","nonce":3630816993}]},"public_addr":"172.21.15.195:6809/3630816993","cluster_addr":"172.21.15.195:6811/3630816993","heartbeat_back_addr":"172.21.15.195:6815/3630816993","heartbeat_front_addr":"172.21.15.195:6813/3630816993","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":43,"last_clean_end":505,"up_from":518,"up_thru":552,"down_at":506,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":977836419},{"type":"v1","addr":"172.21.15.195:6817","nonce":977836419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":977836419},{"type":"v1","addr":"172.21.15.195:6819","nonce":977836419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":977836419},{"type":"v1","addr":"172.21.15.195:6823","nonce":977836419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":977836419},{"type":"v1","addr":"172.21.15.195:6821","nonce":977836419}]},"public_addr":"172.21.15.195:6817/977836419","cluster_addr":"172.21.15.195:6819/977836419","heartbeat_back_addr":"172.21.15.195:6823/977836419","heartbeat_front_addr":"172.21.15.195:6821/977836419","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":49,"last_clean_end":539,"up_from":552,"up_thru":553,"down_at":540,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":72177926},{"type":"v1","addr":"172.21.15.195:6825","nonce":72177926}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":72177926},{"type":"v1","addr":"172.21.15.195:6827","nonce":72177926}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":72177926},{"type":"v1","addr":"172.21.15.195:6831","nonce":72177926}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":72177926},{"type":"v1","addr":"172.21.15.195:6829","nonce":72177926}]},"public_addr":"172.21.15.195:6825/72177926","cluster_addr":"172.21.15.195:6827/72177926","heartbeat_back_addr":"172.21.15.195:6831/72177926","heartbeat_front_addr":"172.21.15.195:6829/72177926","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":311},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":343},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":376},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":409},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":440},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":474},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":505},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":538}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1b","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":2},{"from":6,"to":1}]},{"pgid":"3.20","mappings":[{"from":0,"to":2}]},{"pgid":"3.23","mappings":[{"from":0,"to":2}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.27","mappings":[{"from":4,"to":7}]},{"pgid":"3.30","mappings":[{"from":0,"to":1},{"from":5,"to":2}]},{"pgid":"3.3a","mappings":[{"from":5,"to":7}]},{"pgid":"5.b","mappings":[{"from":0,"to":7}]},{"pgid":"5.e","mappings":[{"from":5,"to":3}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/736862685":"2024-09-15T03:40:41.692273+0000","172.21.15.195:6833/1180864653":"2024-09-15T03:46:16.618713+0000","172.21.15.110:0/2397423297":"2024-09-15T03:40:41.692273+0000","172.21.15.110:0/1810777786":"2024-09-15T03:40:41.692273+0000","172.21.15.110:6800/2030214503":"2024-09-15T03:40:41.692273+0000","172.21.15.110:0/2220961304":"2024-09-15T03:40:41.692273+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/3893271533":"2024-09-15T03:38:59.210484+0000","172.21.15.110:6834/2015946497":"2024-09-15T03:45:58.999716+0000","172.21.15.110:0/974848163":"2024-09-15T03:40:41.692273+0000","172.21.15.110:6801/1862102623":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/2803573586":"2024-09-15T03:40:41.692273+0000","172.21.15.110:0/842858805":"2024-09-15T03:38:59.210484+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2963489706":"2024-09-15T03:38:59.210484+0000","172.21.15.195:0/3089178541":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/1862102623":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/3888509802":"2024-09-15T03:38:59.210484+0000","172.21.15.110:6800/1440218930":"2024-09-15T03:38:59.210484+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2856593023":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/2779556305":"2024-09-15T03:38:59.210484+0000","172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.195:6832/1180864653":"2024-09-15T03:46:16.618713+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.195:0/1517485107":"2024-09-15T03:40:01.254040+0000","172.21.15.195:0/1330315713":"2024-09-15T03:40:01.254040+0000","172.21.15.195:0/2196232450":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/3881395783":"2024-09-15T03:40:41.692273+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.195:0/1384830929":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/4093221761":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6801/1440218930":"2024-09-15T03:38:59.210484+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6801/2030214503":"2024-09-15T03:40:41.692273+0000","172.21.15.195:0/2908653763":"2024-09-15T03:40:01.254040+0000","172.21.15.195:0/145968583":"2024-09-15T03:40:01.254040+0000","172.21.15.110:6835/2015946497":"2024-09-15T03:45:58.999716+0000","172.21.15.195:6834/622119125":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/1769339162":"2024-09-15T03:34:49.809403+0000","172.21.15.195:6835/622119125":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/2214025689":"2024-09-15T03:34:49.809403+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":449,"length":1},{"begin":453,"length":1},{"begin":471,"length":3}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:51:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:33 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2582192374' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T03:51:34.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:33 smithi110 ceph-mon[88262]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T03:51:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:33 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2582192374' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T03:51:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:33 smithi110 ceph-mon[86423]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T03:51:34.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:33 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2582192374' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T03:51:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:33 smithi195 ceph-mon[82388]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T03:51:34.481 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:51:34.728 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:35.077 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:34 smithi110 ceph-mon[88262]: pgmap v950: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 764 B/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-14T03:51:35.078 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:34 smithi110 ceph-mon[86423]: pgmap v950: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 764 B/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-14T03:51:35.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:34 smithi195 ceph-mon[82388]: pgmap v950: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 764 B/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-14T03:51:36.488 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:51:36.489 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2024-09-14T03:51:37.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:36 smithi110 ceph-mon[86423]: pgmap v951: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 765 KiB/s wr, 6 op/s 2024-09-14T03:51:37.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:36 smithi110 ceph-mon[86423]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:51:37.099 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:36 smithi110 ceph-mon[88262]: pgmap v951: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 765 KiB/s wr, 6 op/s 2024-09-14T03:51:37.099 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:36 smithi110 ceph-mon[88262]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:51:37.125 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":951,"stamp":"2024-09-14T03:51:35.866669+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":391027709,"num_objects":166,"num_object_clones":92,"num_object_copies":498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":166,"num_whiteouts":6,"num_read":4674,"num_read_kb":5882685,"num_write":8686,"num_write_kb":2048473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":322,"num_bytes_recovered":606282306,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22287,"ondisk_log_size":22287,"up":363,"acting":363,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3619064,"kb_used_data":616828,"kb_used_omap":328,"kb_used_meta":3001783,"kb_avail":746342152,"statfs":{"total":767960285184,"available":764254363648,"internally_reserved":0,"allocated":631631872,"data_stored":942880901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":336133,"internal_metadata":3073826555},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3751936,"num_objects":2,"num_object_clones":2,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":1,"num_read":4,"num_read_kb":2,"num_write":34,"num_write_kb":4592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001364"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":1150,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.160301+0000","last_change":"2024-09-14T03:45:54.804965+0000","last_active":"2024-09-14T03:51:32.160301+0000","last_peered":"2024-09-14T03:51:32.160301+0000","last_clean":"2024-09-14T03:51:32.160301+0000","last_became_active":"2024-09-14T03:45:54.804696+0000","last_became_peered":"2024-09-14T03:45:54.804696+0000","last_unstale":"2024-09-14T03:51:32.160301+0000","last_undegraded":"2024-09-14T03:51:32.160301+0000","last_fullsized":"2024-09-14T03:51:32.160301+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:42.590664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":1348,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855265+0000","last_change":"2024-09-14T03:45:20.807202+0000","last_active":"2024-09-14T03:51:31.855265+0000","last_peered":"2024-09-14T03:51:31.855265+0000","last_clean":"2024-09-14T03:51:31.855265+0000","last_became_active":"2024-09-14T03:45:20.806886+0000","last_became_peered":"2024-09-14T03:45:20.806886+0000","last_unstale":"2024-09-14T03:51:31.855265+0000","last_undegraded":"2024-09-14T03:51:31.855265+0000","last_fullsized":"2024-09-14T03:51:31.855265+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:40.778025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":1386,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026704+0000","last_change":"2024-09-14T03:43:06.751752+0000","last_active":"2024-09-14T03:51:34.026704+0000","last_peered":"2024-09-14T03:51:34.026704+0000","last_clean":"2024-09-14T03:51:34.026704+0000","last_became_active":"2024-09-14T03:43:06.751407+0000","last_became_peered":"2024-09-14T03:43:06.751407+0000","last_unstale":"2024-09-14T03:51:34.026704+0000","last_undegraded":"2024-09-14T03:51:34.026704+0000","last_fullsized":"2024-09-14T03:51:34.026704+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:18.018893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":1224,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.325750+0000","last_change":"2024-09-14T03:45:54.804964+0000","last_active":"2024-09-14T03:51:34.325750+0000","last_peered":"2024-09-14T03:51:34.325750+0000","last_clean":"2024-09-14T03:51:34.325750+0000","last_became_active":"2024-09-14T03:45:54.804125+0000","last_became_peered":"2024-09-14T03:45:54.804125+0000","last_unstale":"2024-09-14T03:51:34.325750+0000","last_undegraded":"2024-09-14T03:51:34.325750+0000","last_fullsized":"2024-09-14T03:51:34.325750+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:06.572323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":1169,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159545+0000","last_change":"2024-09-14T03:45:54.814740+0000","last_active":"2024-09-14T03:51:32.159545+0000","last_peered":"2024-09-14T03:51:32.159545+0000","last_clean":"2024-09-14T03:51:32.159545+0000","last_became_active":"2024-09-14T03:45:54.812166+0000","last_became_peered":"2024-09-14T03:45:54.812166+0000","last_unstale":"2024-09-14T03:51:32.159545+0000","last_undegraded":"2024-09-14T03:51:32.159545+0000","last_fullsized":"2024-09-14T03:51:32.159545+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:59:45.607099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":650,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159205+0000","last_change":"2024-09-14T03:45:55.801630+0000","last_active":"2024-09-14T03:51:32.159205+0000","last_peered":"2024-09-14T03:51:32.159205+0000","last_clean":"2024-09-14T03:51:32.159205+0000","last_became_active":"2024-09-14T03:45:55.801256+0000","last_became_peered":"2024-09-14T03:45:55.801256+0000","last_unstale":"2024-09-14T03:51:32.159205+0000","last_undegraded":"2024-09-14T03:51:32.159205+0000","last_fullsized":"2024-09-14T03:51:32.159205+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:07.768236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":1169,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159133+0000","last_change":"2024-09-14T03:45:54.803820+0000","last_active":"2024-09-14T03:51:32.159133+0000","last_peered":"2024-09-14T03:51:32.159133+0000","last_clean":"2024-09-14T03:51:32.159133+0000","last_became_active":"2024-09-14T03:45:54.803472+0000","last_became_peered":"2024-09-14T03:45:54.803472+0000","last_unstale":"2024-09-14T03:51:32.159133+0000","last_undegraded":"2024-09-14T03:51:32.159133+0000","last_fullsized":"2024-09-14T03:51:32.159133+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:54.989956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":1273,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.860737+0000","last_change":"2024-09-14T03:45:20.804182+0000","last_active":"2024-09-14T03:51:31.860737+0000","last_peered":"2024-09-14T03:51:31.860737+0000","last_clean":"2024-09-14T03:51:31.860737+0000","last_became_active":"2024-09-14T03:45:20.803817+0000","last_became_peered":"2024-09-14T03:45:20.803817+0000","last_unstale":"2024-09-14T03:51:31.860737+0000","last_undegraded":"2024-09-14T03:51:31.860737+0000","last_fullsized":"2024-09-14T03:51:31.860737+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:12:29.632073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":1160,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.160447+0000","last_change":"2024-09-14T03:45:54.811459+0000","last_active":"2024-09-14T03:51:32.160447+0000","last_peered":"2024-09-14T03:51:32.160447+0000","last_clean":"2024-09-14T03:51:32.160447+0000","last_became_active":"2024-09-14T03:45:54.808650+0000","last_became_peered":"2024-09-14T03:45:54.808650+0000","last_unstale":"2024-09-14T03:51:32.160447+0000","last_undegraded":"2024-09-14T03:51:32.160447+0000","last_fullsized":"2024-09-14T03:51:32.160447+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:02.073671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":1187,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855352+0000","last_change":"2024-09-14T03:45:20.812467+0000","last_active":"2024-09-14T03:51:31.855352+0000","last_peered":"2024-09-14T03:51:31.855352+0000","last_clean":"2024-09-14T03:51:31.855352+0000","last_became_active":"2024-09-14T03:45:20.812213+0000","last_became_peered":"2024-09-14T03:45:20.812213+0000","last_unstale":"2024-09-14T03:51:31.855352+0000","last_undegraded":"2024-09-14T03:51:31.855352+0000","last_fullsized":"2024-09-14T03:51:31.855352+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:27:16.357609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":1187,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.854828+0000","last_change":"2024-09-14T03:45:20.807117+0000","last_active":"2024-09-14T03:51:31.854828+0000","last_peered":"2024-09-14T03:51:31.854828+0000","last_clean":"2024-09-14T03:51:31.854828+0000","last_became_active":"2024-09-14T03:45:20.806386+0000","last_became_peered":"2024-09-14T03:45:20.806386+0000","last_unstale":"2024-09-14T03:51:31.854828+0000","last_undegraded":"2024-09-14T03:51:31.854828+0000","last_fullsized":"2024-09-14T03:51:31.854828+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:02:44.514940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":1237,"reported_epoch":873,"state":"active+clean","last_fresh":"2024-09-14T03:51:30.852900+0000","last_change":"2024-09-14T03:45:20.812750+0000","last_active":"2024-09-14T03:51:30.852900+0000","last_peered":"2024-09-14T03:51:30.852900+0000","last_clean":"2024-09-14T03:51:30.852900+0000","last_became_active":"2024-09-14T03:45:20.812494+0000","last_became_peered":"2024-09-14T03:45:20.812494+0000","last_unstale":"2024-09-14T03:51:30.852900+0000","last_undegraded":"2024-09-14T03:51:30.852900+0000","last_fullsized":"2024-09-14T03:51:30.852900+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:52:59.776669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":1295,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861022+0000","last_change":"2024-09-14T03:45:20.804912+0000","last_active":"2024-09-14T03:51:31.861022+0000","last_peered":"2024-09-14T03:51:31.861022+0000","last_clean":"2024-09-14T03:51:31.861022+0000","last_became_active":"2024-09-14T03:45:20.804209+0000","last_became_peered":"2024-09-14T03:45:20.804209+0000","last_unstale":"2024-09-14T03:51:31.861022+0000","last_undegraded":"2024-09-14T03:51:31.861022+0000","last_fullsized":"2024-09-14T03:51:31.861022+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:13.279472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":1386,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026371+0000","last_change":"2024-09-14T03:44:47.747016+0000","last_active":"2024-09-14T03:51:34.026371+0000","last_peered":"2024-09-14T03:51:34.026371+0000","last_clean":"2024-09-14T03:51:34.026371+0000","last_became_active":"2024-09-14T03:44:47.746726+0000","last_became_peered":"2024-09-14T03:44:47.746726+0000","last_unstale":"2024-09-14T03:51:34.026371+0000","last_undegraded":"2024-09-14T03:51:34.026371+0000","last_fullsized":"2024-09-14T03:51:34.026371+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:20.445329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":1359,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855188+0000","last_change":"2024-09-14T03:44:47.744583+0000","last_active":"2024-09-14T03:51:31.855188+0000","last_peered":"2024-09-14T03:51:31.855188+0000","last_clean":"2024-09-14T03:51:31.855188+0000","last_became_active":"2024-09-14T03:44:47.744267+0000","last_became_peered":"2024-09-14T03:44:47.744267+0000","last_unstale":"2024-09-14T03:51:31.855188+0000","last_undegraded":"2024-09-14T03:51:31.855188+0000","last_fullsized":"2024-09-14T03:51:31.855188+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:44.517725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":684,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.846705+0000","last_change":"2024-09-14T03:45:55.795774+0000","last_active":"2024-09-14T03:51:33.846705+0000","last_peered":"2024-09-14T03:51:33.846705+0000","last_clean":"2024-09-14T03:51:33.846705+0000","last_became_active":"2024-09-14T03:45:55.794057+0000","last_became_peered":"2024-09-14T03:45:55.794057+0000","last_unstale":"2024-09-14T03:51:33.846705+0000","last_undegraded":"2024-09-14T03:51:33.846705+0000","last_fullsized":"2024-09-14T03:51:33.846705+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:45.746306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":1215,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.326877+0000","last_change":"2024-09-14T03:45:54.806054+0000","last_active":"2024-09-14T03:51:34.326877+0000","last_peered":"2024-09-14T03:51:34.326877+0000","last_clean":"2024-09-14T03:51:34.326877+0000","last_became_active":"2024-09-14T03:45:54.805712+0000","last_became_peered":"2024-09-14T03:45:54.805712+0000","last_unstale":"2024-09-14T03:51:34.326877+0000","last_undegraded":"2024-09-14T03:51:34.326877+0000","last_fullsized":"2024-09-14T03:51:34.326877+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:26.749011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":1297,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.848096+0000","last_change":"2024-09-14T03:45:54.813118+0000","last_active":"2024-09-14T03:51:33.848096+0000","last_peered":"2024-09-14T03:51:33.848096+0000","last_clean":"2024-09-14T03:51:33.848096+0000","last_became_active":"2024-09-14T03:45:54.812730+0000","last_became_peered":"2024-09-14T03:45:54.812730+0000","last_unstale":"2024-09-14T03:51:33.848096+0000","last_undegraded":"2024-09-14T03:51:33.848096+0000","last_fullsized":"2024-09-14T03:51:33.848096+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:52:45.371660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":1150,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.160002+0000","last_change":"2024-09-14T03:45:54.816330+0000","last_active":"2024-09-14T03:51:32.160002+0000","last_peered":"2024-09-14T03:51:32.160002+0000","last_clean":"2024-09-14T03:51:32.160002+0000","last_became_active":"2024-09-14T03:45:54.816069+0000","last_became_peered":"2024-09-14T03:45:54.816069+0000","last_unstale":"2024-09-14T03:51:32.160002+0000","last_undegraded":"2024-09-14T03:51:32.160002+0000","last_fullsized":"2024-09-14T03:51:32.160002+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:59.705720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":1197,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.854781+0000","last_change":"2024-09-14T03:45:20.810439+0000","last_active":"2024-09-14T03:51:31.854781+0000","last_peered":"2024-09-14T03:51:31.854781+0000","last_clean":"2024-09-14T03:51:31.854781+0000","last_became_active":"2024-09-14T03:45:20.809992+0000","last_became_peered":"2024-09-14T03:45:20.809992+0000","last_unstale":"2024-09-14T03:51:31.854781+0000","last_undegraded":"2024-09-14T03:51:31.854781+0000","last_fullsized":"2024-09-14T03:51:31.854781+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:09.152978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":1386,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026515+0000","last_change":"2024-09-14T03:45:54.806691+0000","last_active":"2024-09-14T03:51:34.026515+0000","last_peered":"2024-09-14T03:51:34.026515+0000","last_clean":"2024-09-14T03:51:34.026515+0000","last_became_active":"2024-09-14T03:45:54.805713+0000","last_became_peered":"2024-09-14T03:45:54.805713+0000","last_unstale":"2024-09-14T03:51:34.026515+0000","last_undegraded":"2024-09-14T03:51:34.026515+0000","last_fullsized":"2024-09-14T03:51:34.026515+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:43:50.369795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":1222,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.325724+0000","last_change":"2024-09-14T03:45:54.797239+0000","last_active":"2024-09-14T03:51:34.325724+0000","last_peered":"2024-09-14T03:51:34.325724+0000","last_clean":"2024-09-14T03:51:34.325724+0000","last_became_active":"2024-09-14T03:45:54.796929+0000","last_became_peered":"2024-09-14T03:45:54.796929+0000","last_unstale":"2024-09-14T03:51:34.325724+0000","last_undegraded":"2024-09-14T03:51:34.325724+0000","last_fullsized":"2024-09-14T03:51:34.325724+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:45.249847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":1295,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861110+0000","last_change":"2024-09-14T03:44:13.747197+0000","last_active":"2024-09-14T03:51:31.861110+0000","last_peered":"2024-09-14T03:51:31.861110+0000","last_clean":"2024-09-14T03:51:31.861110+0000","last_became_active":"2024-09-14T03:44:13.746898+0000","last_became_peered":"2024-09-14T03:44:13.746898+0000","last_unstale":"2024-09-14T03:51:31.861110+0000","last_undegraded":"2024-09-14T03:51:31.861110+0000","last_fullsized":"2024-09-14T03:51:31.861110+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:24.954597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":1230,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.325225+0000","last_change":"2024-09-14T03:44:47.747383+0000","last_active":"2024-09-14T03:51:34.325225+0000","last_peered":"2024-09-14T03:51:34.325225+0000","last_clean":"2024-09-14T03:51:34.325225+0000","last_became_active":"2024-09-14T03:44:47.746390+0000","last_became_peered":"2024-09-14T03:44:47.746390+0000","last_unstale":"2024-09-14T03:51:34.325225+0000","last_undegraded":"2024-09-14T03:51:34.325225+0000","last_fullsized":"2024-09-14T03:51:34.325225+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:13.437711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":1296,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861759+0000","last_change":"2024-09-14T03:45:55.798090+0000","last_active":"2024-09-14T03:51:31.861759+0000","last_peered":"2024-09-14T03:51:31.861759+0000","last_clean":"2024-09-14T03:51:31.861759+0000","last_became_active":"2024-09-14T03:45:55.797554+0000","last_became_peered":"2024-09-14T03:45:55.797554+0000","last_unstale":"2024-09-14T03:51:31.861759+0000","last_undegraded":"2024-09-14T03:51:31.861759+0000","last_fullsized":"2024-09-14T03:51:31.861759+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:06.225750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":1191,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855089+0000","last_change":"2024-09-14T03:45:54.813493+0000","last_active":"2024-09-14T03:51:31.855089+0000","last_peered":"2024-09-14T03:51:31.855089+0000","last_clean":"2024-09-14T03:51:31.855089+0000","last_became_active":"2024-09-14T03:45:54.813179+0000","last_became_peered":"2024-09-14T03:45:54.813179+0000","last_unstale":"2024-09-14T03:51:31.855089+0000","last_undegraded":"2024-09-14T03:51:31.855089+0000","last_fullsized":"2024-09-14T03:51:31.855089+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:41.465829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":1360,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855187+0000","last_change":"2024-09-14T03:43:39.739966+0000","last_active":"2024-09-14T03:51:31.855187+0000","last_peered":"2024-09-14T03:51:31.855187+0000","last_clean":"2024-09-14T03:51:31.855187+0000","last_became_active":"2024-09-14T03:43:39.739688+0000","last_became_peered":"2024-09-14T03:43:39.739688+0000","last_unstale":"2024-09-14T03:51:31.855187+0000","last_undegraded":"2024-09-14T03:51:31.855187+0000","last_fullsized":"2024-09-14T03:51:31.855187+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:12:30.033160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":1300,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861487+0000","last_change":"2024-09-14T03:43:39.739024+0000","last_active":"2024-09-14T03:51:31.861487+0000","last_peered":"2024-09-14T03:51:31.861487+0000","last_clean":"2024-09-14T03:51:31.861487+0000","last_became_active":"2024-09-14T03:43:39.737791+0000","last_became_peered":"2024-09-14T03:43:39.737791+0000","last_unstale":"2024-09-14T03:51:31.861487+0000","last_undegraded":"2024-09-14T03:51:31.861487+0000","last_fullsized":"2024-09-14T03:51:31.861487+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:30:32.849662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":1300,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.847523+0000","last_change":"2024-09-14T03:44:13.744001+0000","last_active":"2024-09-14T03:51:33.847523+0000","last_peered":"2024-09-14T03:51:33.847523+0000","last_clean":"2024-09-14T03:51:33.847523+0000","last_became_active":"2024-09-14T03:44:13.743702+0000","last_became_peered":"2024-09-14T03:44:13.743702+0000","last_unstale":"2024-09-14T03:51:33.847523+0000","last_undegraded":"2024-09-14T03:51:33.847523+0000","last_fullsized":"2024-09-14T03:51:33.847523+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:37:34.963319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":1230,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.325832+0000","last_change":"2024-09-14T03:44:47.744917+0000","last_active":"2024-09-14T03:51:34.325832+0000","last_peered":"2024-09-14T03:51:34.325832+0000","last_clean":"2024-09-14T03:51:34.325832+0000","last_became_active":"2024-09-14T03:44:47.744429+0000","last_became_peered":"2024-09-14T03:44:47.744429+0000","last_unstale":"2024-09-14T03:51:34.325832+0000","last_undegraded":"2024-09-14T03:51:34.325832+0000","last_fullsized":"2024-09-14T03:51:34.325832+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:42:43.885114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":1298,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.846780+0000","last_change":"2024-09-14T03:45:20.797579+0000","last_active":"2024-09-14T03:51:33.846780+0000","last_peered":"2024-09-14T03:51:33.846780+0000","last_clean":"2024-09-14T03:51:33.846780+0000","last_became_active":"2024-09-14T03:45:20.797232+0000","last_became_peered":"2024-09-14T03:45:20.797232+0000","last_unstale":"2024-09-14T03:51:33.846780+0000","last_undegraded":"2024-09-14T03:51:33.846780+0000","last_fullsized":"2024-09-14T03:51:33.846780+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:47:09.878505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":1195,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.854442+0000","last_change":"2024-09-14T03:45:43.744248+0000","last_active":"2024-09-14T03:51:31.854442+0000","last_peered":"2024-09-14T03:51:31.854442+0000","last_clean":"2024-09-14T03:51:31.854442+0000","last_became_active":"2024-09-14T03:45:43.743909+0000","last_became_peered":"2024-09-14T03:45:43.743909+0000","last_unstale":"2024-09-14T03:51:31.854442+0000","last_undegraded":"2024-09-14T03:51:31.854442+0000","last_fullsized":"2024-09-14T03:51:31.854442+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:44:41.712704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":1265,"reported_epoch":873,"state":"active+clean","last_fresh":"2024-09-14T03:51:30.852540+0000","last_change":"2024-09-14T03:44:13.736938+0000","last_active":"2024-09-14T03:51:30.852540+0000","last_peered":"2024-09-14T03:51:30.852540+0000","last_clean":"2024-09-14T03:51:30.852540+0000","last_became_active":"2024-09-14T03:44:13.736660+0000","last_became_peered":"2024-09-14T03:44:13.736660+0000","last_unstale":"2024-09-14T03:51:30.852540+0000","last_undegraded":"2024-09-14T03:51:30.852540+0000","last_fullsized":"2024-09-14T03:51:30.852540+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:35:08.636827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"576'3","reported_seq":1288,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861049+0000","last_change":"2024-09-14T03:46:44.764394+0000","last_active":"2024-09-14T03:51:31.861049+0000","last_peered":"2024-09-14T03:51:31.861049+0000","last_clean":"2024-09-14T03:51:31.861049+0000","last_became_active":"2024-09-14T03:46:44.763888+0000","last_became_peered":"2024-09-14T03:46:44.763888+0000","last_unstale":"2024-09-14T03:51:31.861049+0000","last_undegraded":"2024-09-14T03:51:31.861049+0000","last_fullsized":"2024-09-14T03:51:31.861049+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:03:54.910205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":1385,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026601+0000","last_change":"2024-09-14T03:45:54.807029+0000","last_active":"2024-09-14T03:51:34.026601+0000","last_peered":"2024-09-14T03:51:34.026601+0000","last_clean":"2024-09-14T03:51:34.026601+0000","last_became_active":"2024-09-14T03:45:54.806709+0000","last_became_peered":"2024-09-14T03:45:54.806709+0000","last_unstale":"2024-09-14T03:51:34.026601+0000","last_undegraded":"2024-09-14T03:51:34.026601+0000","last_fullsized":"2024-09-14T03:51:34.026601+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:51:33.720783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"576'3","reported_seq":1197,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.854989+0000","last_change":"2024-09-14T03:45:54.801497+0000","last_active":"2024-09-14T03:51:31.854989+0000","last_peered":"2024-09-14T03:51:31.854989+0000","last_clean":"2024-09-14T03:51:31.854989+0000","last_became_active":"2024-09-14T03:45:54.801004+0000","last_became_peered":"2024-09-14T03:45:54.801004+0000","last_unstale":"2024-09-14T03:51:31.854989+0000","last_undegraded":"2024-09-14T03:51:31.854989+0000","last_fullsized":"2024-09-14T03:51:31.854989+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:56.014624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":1305,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.860737+0000","last_change":"2024-09-14T03:45:20.806357+0000","last_active":"2024-09-14T03:51:31.860737+0000","last_peered":"2024-09-14T03:51:31.860737+0000","last_clean":"2024-09-14T03:51:31.860737+0000","last_became_active":"2024-09-14T03:45:20.804994+0000","last_became_peered":"2024-09-14T03:45:20.804994+0000","last_unstale":"2024-09-14T03:51:31.860737+0000","last_undegraded":"2024-09-14T03:51:31.860737+0000","last_fullsized":"2024-09-14T03:51:31.860737+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:12.718545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1301,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.860850+0000","last_change":"2024-09-14T03:43:39.735770+0000","last_active":"2024-09-14T03:51:31.860850+0000","last_peered":"2024-09-14T03:51:31.860850+0000","last_clean":"2024-09-14T03:51:31.860850+0000","last_became_active":"2024-09-14T03:43:39.735197+0000","last_became_peered":"2024-09-14T03:43:39.735197+0000","last_unstale":"2024-09-14T03:51:31.860850+0000","last_undegraded":"2024-09-14T03:51:31.860850+0000","last_fullsized":"2024-09-14T03:51:31.860850+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:50.053746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":1218,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.325102+0000","last_change":"2024-09-14T03:45:54.804534+0000","last_active":"2024-09-14T03:51:34.325102+0000","last_peered":"2024-09-14T03:51:34.325102+0000","last_clean":"2024-09-14T03:51:34.325102+0000","last_became_active":"2024-09-14T03:45:54.804212+0000","last_became_peered":"2024-09-14T03:45:54.804212+0000","last_unstale":"2024-09-14T03:51:34.325102+0000","last_undegraded":"2024-09-14T03:51:34.325102+0000","last_fullsized":"2024-09-14T03:51:34.325102+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:06.056738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1359,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855428+0000","last_change":"2024-09-14T03:44:13.732516+0000","last_active":"2024-09-14T03:51:31.855428+0000","last_peered":"2024-09-14T03:51:31.855428+0000","last_clean":"2024-09-14T03:51:31.855428+0000","last_became_active":"2024-09-14T03:44:13.732228+0000","last_became_peered":"2024-09-14T03:44:13.732228+0000","last_unstale":"2024-09-14T03:51:31.855428+0000","last_undegraded":"2024-09-14T03:51:31.855428+0000","last_fullsized":"2024-09-14T03:51:31.855428+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:17:20.673681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":1190,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855246+0000","last_change":"2024-09-14T03:45:20.805335+0000","last_active":"2024-09-14T03:51:31.855246+0000","last_peered":"2024-09-14T03:51:31.855246+0000","last_clean":"2024-09-14T03:51:31.855246+0000","last_became_active":"2024-09-14T03:45:20.804471+0000","last_became_peered":"2024-09-14T03:45:20.804471+0000","last_unstale":"2024-09-14T03:51:31.855246+0000","last_undegraded":"2024-09-14T03:51:31.855246+0000","last_fullsized":"2024-09-14T03:51:31.855246+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:37:33.284325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1202,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.326775+0000","last_change":"2024-09-14T03:45:54.816740+0000","last_active":"2024-09-14T03:51:34.326775+0000","last_peered":"2024-09-14T03:51:34.326775+0000","last_clean":"2024-09-14T03:51:34.326775+0000","last_became_active":"2024-09-14T03:45:54.816467+0000","last_became_peered":"2024-09-14T03:45:54.816467+0000","last_unstale":"2024-09-14T03:51:34.326775+0000","last_undegraded":"2024-09-14T03:51:34.326775+0000","last_fullsized":"2024-09-14T03:51:34.326775+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:38.779221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":1236,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.326882+0000","last_change":"2024-09-14T03:45:20.798201+0000","last_active":"2024-09-14T03:51:34.326882+0000","last_peered":"2024-09-14T03:51:34.326882+0000","last_clean":"2024-09-14T03:51:34.326882+0000","last_became_active":"2024-09-14T03:45:20.797891+0000","last_became_peered":"2024-09-14T03:45:20.797891+0000","last_unstale":"2024-09-14T03:51:34.326882+0000","last_undegraded":"2024-09-14T03:51:34.326882+0000","last_fullsized":"2024-09-14T03:51:34.326882+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:49:46.807662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1385,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026781+0000","last_change":"2024-09-14T03:44:47.742400+0000","last_active":"2024-09-14T03:51:34.026781+0000","last_peered":"2024-09-14T03:51:34.026781+0000","last_clean":"2024-09-14T03:51:34.026781+0000","last_became_active":"2024-09-14T03:44:47.742079+0000","last_became_peered":"2024-09-14T03:44:47.742079+0000","last_unstale":"2024-09-14T03:51:34.026781+0000","last_undegraded":"2024-09-14T03:51:34.026781+0000","last_fullsized":"2024-09-14T03:51:34.026781+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:02:31.101033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":1358,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855487+0000","last_change":"2024-09-14T03:44:47.745389+0000","last_active":"2024-09-14T03:51:31.855487+0000","last_peered":"2024-09-14T03:51:31.855487+0000","last_clean":"2024-09-14T03:51:31.855487+0000","last_became_active":"2024-09-14T03:44:47.744962+0000","last_became_peered":"2024-09-14T03:44:47.744962+0000","last_unstale":"2024-09-14T03:51:31.855487+0000","last_undegraded":"2024-09-14T03:51:31.855487+0000","last_fullsized":"2024-09-14T03:51:31.855487+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:54.227233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":11,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":1299,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.847119+0000","last_change":"2024-09-14T03:45:55.795990+0000","last_active":"2024-09-14T03:51:33.847119+0000","last_peered":"2024-09-14T03:51:33.847119+0000","last_clean":"2024-09-14T03:51:33.847119+0000","last_became_active":"2024-09-14T03:45:55.794070+0000","last_became_peered":"2024-09-14T03:45:55.794070+0000","last_unstale":"2024-09-14T03:51:33.847119+0000","last_undegraded":"2024-09-14T03:51:33.847119+0000","last_fullsized":"2024-09-14T03:51:33.847119+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:31.868540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.14","version":"576'30","reported_seq":1231,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.854971+0000","last_change":"2024-09-14T03:45:20.805168+0000","last_active":"2024-09-14T03:51:31.854971+0000","last_peered":"2024-09-14T03:51:31.854971+0000","last_clean":"2024-09-14T03:51:31.854971+0000","last_became_active":"2024-09-14T03:45:20.804820+0000","last_became_peered":"2024-09-14T03:45:20.804820+0000","last_unstale":"2024-09-14T03:51:31.854971+0000","last_undegraded":"2024-09-14T03:51:31.854971+0000","last_fullsized":"2024-09-14T03:51:31.854971+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:52.802483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5816,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":17,"num_read_kb":25,"num_write":28,"num_write_kb":25,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":1169,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.160097+0000","last_change":"2024-09-14T03:45:54.814764+0000","last_active":"2024-09-14T03:51:32.160097+0000","last_peered":"2024-09-14T03:51:32.160097+0000","last_clean":"2024-09-14T03:51:32.160097+0000","last_became_active":"2024-09-14T03:45:54.814393+0000","last_became_peered":"2024-09-14T03:45:54.814393+0000","last_unstale":"2024-09-14T03:51:32.160097+0000","last_undegraded":"2024-09-14T03:51:32.160097+0000","last_fullsized":"2024-09-14T03:51:32.160097+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:50.317109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":558,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026371+0000","last_change":"2024-09-14T03:46:44.767731+0000","last_active":"2024-09-14T03:51:34.026371+0000","last_peered":"2024-09-14T03:51:34.026371+0000","last_clean":"2024-09-14T03:51:34.026371+0000","last_became_active":"2024-09-14T03:46:44.763804+0000","last_became_peered":"2024-09-14T03:46:44.763804+0000","last_unstale":"2024-09-14T03:51:34.026371+0000","last_undegraded":"2024-09-14T03:51:34.026371+0000","last_fullsized":"2024-09-14T03:51:34.026371+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:29:33.866379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1386,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026317+0000","last_change":"2024-09-14T03:45:54.798239+0000","last_active":"2024-09-14T03:51:34.026317+0000","last_peered":"2024-09-14T03:51:34.026317+0000","last_clean":"2024-09-14T03:51:34.026317+0000","last_became_active":"2024-09-14T03:45:54.797922+0000","last_became_peered":"2024-09-14T03:45:54.797922+0000","last_unstale":"2024-09-14T03:51:34.026317+0000","last_undegraded":"2024-09-14T03:51:34.026317+0000","last_fullsized":"2024-09-14T03:51:34.026317+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:48:10.868912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":1226,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.326112+0000","last_change":"2024-09-14T03:45:54.798363+0000","last_active":"2024-09-14T03:51:34.326112+0000","last_peered":"2024-09-14T03:51:34.326112+0000","last_clean":"2024-09-14T03:51:34.326112+0000","last_became_active":"2024-09-14T03:45:54.798061+0000","last_became_peered":"2024-09-14T03:45:54.798061+0000","last_unstale":"2024-09-14T03:51:34.326112+0000","last_undegraded":"2024-09-14T03:51:34.326112+0000","last_fullsized":"2024-09-14T03:51:34.326112+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:28.120868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":1169,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159419+0000","last_change":"2024-09-14T03:45:54.814379+0000","last_active":"2024-09-14T03:51:32.159419+0000","last_peered":"2024-09-14T03:51:32.159419+0000","last_clean":"2024-09-14T03:51:32.159419+0000","last_became_active":"2024-09-14T03:45:54.811828+0000","last_became_peered":"2024-09-14T03:45:54.811828+0000","last_unstale":"2024-09-14T03:51:32.159419+0000","last_undegraded":"2024-09-14T03:51:32.159419+0000","last_fullsized":"2024-09-14T03:51:32.159419+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:21.512082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":1200,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.854570+0000","last_change":"2024-09-14T03:45:20.811935+0000","last_active":"2024-09-14T03:51:31.854570+0000","last_peered":"2024-09-14T03:51:31.854570+0000","last_clean":"2024-09-14T03:51:31.854570+0000","last_became_active":"2024-09-14T03:45:20.808783+0000","last_became_peered":"2024-09-14T03:45:20.808783+0000","last_unstale":"2024-09-14T03:51:31.854570+0000","last_undegraded":"2024-09-14T03:51:31.854570+0000","last_fullsized":"2024-09-14T03:51:31.854570+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:58:25.597782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":1187,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.854402+0000","last_change":"2024-09-14T03:45:20.809262+0000","last_active":"2024-09-14T03:51:31.854402+0000","last_peered":"2024-09-14T03:51:31.854402+0000","last_clean":"2024-09-14T03:51:31.854402+0000","last_became_active":"2024-09-14T03:45:20.807994+0000","last_became_peered":"2024-09-14T03:45:20.807994+0000","last_unstale":"2024-09-14T03:51:31.854402+0000","last_undegraded":"2024-09-14T03:51:31.854402+0000","last_fullsized":"2024-09-14T03:51:31.854402+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:40:47.463485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":1347,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.846828+0000","last_change":"2024-09-14T03:43:06.751094+0000","last_active":"2024-09-14T03:51:33.846828+0000","last_peered":"2024-09-14T03:51:33.846828+0000","last_clean":"2024-09-14T03:51:33.846828+0000","last_became_active":"2024-09-14T03:43:06.750480+0000","last_became_peered":"2024-09-14T03:43:06.750480+0000","last_unstale":"2024-09-14T03:51:33.846828+0000","last_undegraded":"2024-09-14T03:51:33.846828+0000","last_fullsized":"2024-09-14T03:51:33.846828+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:58:59.077750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":36,"num_read_kb":36,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1150,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159025+0000","last_change":"2024-09-14T03:45:54.815991+0000","last_active":"2024-09-14T03:51:32.159025+0000","last_peered":"2024-09-14T03:51:32.159025+0000","last_clean":"2024-09-14T03:51:32.159025+0000","last_became_active":"2024-09-14T03:45:54.814964+0000","last_became_peered":"2024-09-14T03:45:54.814964+0000","last_unstale":"2024-09-14T03:51:32.159025+0000","last_undegraded":"2024-09-14T03:51:32.159025+0000","last_fullsized":"2024-09-14T03:51:32.159025+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:43:13.292700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"865'464","reported_seq":3927,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.846965+0000","last_change":"2024-09-14T03:51:29.867391+0000","last_active":"2024-09-14T03:51:33.846965+0000","last_peered":"2024-09-14T03:51:33.846965+0000","last_clean":"2024-09-14T03:51:33.846965+0000","last_became_active":"2024-09-14T03:43:06.752953+0000","last_became_peered":"2024-09-14T03:43:06.752953+0000","last_unstale":"2024-09-14T03:51:33.846965+0000","last_undegraded":"2024-09-14T03:51:33.846965+0000","last_fullsized":"2024-09-14T03:51:33.846965+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":464,"log_dups_size":0,"ondisk_log_size":464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:12.582102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012930660000000001,"stat_sum":{"num_bytes":10059776,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":74,"num_read_kb":244284,"num_write":88,"num_write_kb":54936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":13238272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"2.f","version":"58'1","reported_seq":1243,"reported_epoch":873,"state":"active+clean","last_fresh":"2024-09-14T03:51:30.853066+0000","last_change":"2024-09-14T03:45:54.805574+0000","last_active":"2024-09-14T03:51:30.853066+0000","last_peered":"2024-09-14T03:51:30.853066+0000","last_clean":"2024-09-14T03:51:30.853066+0000","last_became_active":"2024-09-14T03:45:54.805218+0000","last_became_peered":"2024-09-14T03:45:54.805218+0000","last_unstale":"2024-09-14T03:51:30.853066+0000","last_undegraded":"2024-09-14T03:51:30.853066+0000","last_fullsized":"2024-09-14T03:51:30.853066+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:07.008099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":1169,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.160055+0000","last_change":"2024-09-14T03:45:54.811640+0000","last_active":"2024-09-14T03:51:32.160055+0000","last_peered":"2024-09-14T03:51:32.160055+0000","last_clean":"2024-09-14T03:51:32.160055+0000","last_became_active":"2024-09-14T03:45:54.810960+0000","last_became_peered":"2024-09-14T03:45:54.810960+0000","last_unstale":"2024-09-14T03:51:32.160055+0000","last_undegraded":"2024-09-14T03:51:32.160055+0000","last_fullsized":"2024-09-14T03:51:32.160055+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:43:56.926229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"872'1636","reported_seq":4660,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159026+0000","last_change":"2024-09-14T03:51:29.878002+0000","last_active":"2024-09-14T03:51:32.159026+0000","last_peered":"2024-09-14T03:51:32.159026+0000","last_clean":"2024-09-14T03:51:32.159026+0000","last_became_active":"2024-09-14T03:45:54.807906+0000","last_became_peered":"2024-09-14T03:45:54.807906+0000","last_unstale":"2024-09-14T03:51:32.159026+0000","last_undegraded":"2024-09-14T03:51:32.159026+0000","last_fullsized":"2024-09-14T03:51:32.159026+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":1636,"log_dups_size":0,"ondisk_log_size":1636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:42.851842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0059485129999999999,"stat_sum":{"num_bytes":19152896,"num_objects":9,"num_object_clones":6,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":242,"num_read_kb":280944,"num_write":740,"num_write_kb":158624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":24084480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1c1","length":"1"},{"start":"1c5","length":"1"},{"start":"1d7","length":"3"}]},{"pgid":"2.e","version":"58'2","reported_seq":1307,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.847694+0000","last_change":"2024-09-14T03:45:54.813111+0000","last_active":"2024-09-14T03:51:33.847694+0000","last_peered":"2024-09-14T03:51:33.847694+0000","last_clean":"2024-09-14T03:51:33.847694+0000","last_became_active":"2024-09-14T03:45:54.812715+0000","last_became_peered":"2024-09-14T03:45:54.812715+0000","last_unstale":"2024-09-14T03:51:33.847694+0000","last_undegraded":"2024-09-14T03:51:33.847694+0000","last_fullsized":"2024-09-14T03:51:33.847694+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:01.594515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":9,"num_read_kb":9,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":1169,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.160436+0000","last_change":"2024-09-14T03:45:54.812182+0000","last_active":"2024-09-14T03:51:32.160436+0000","last_peered":"2024-09-14T03:51:32.160436+0000","last_clean":"2024-09-14T03:51:32.160436+0000","last_became_active":"2024-09-14T03:45:54.811873+0000","last_became_peered":"2024-09-14T03:45:54.811873+0000","last_unstale":"2024-09-14T03:51:32.160436+0000","last_undegraded":"2024-09-14T03:51:32.160436+0000","last_fullsized":"2024-09-14T03:51:32.160436+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:00:43.919679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.a","version":"874'2322","reported_seq":5607,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.847346+0000","last_change":"2024-09-14T03:51:29.877987+0000","last_active":"2024-09-14T03:51:33.847346+0000","last_peered":"2024-09-14T03:51:33.847346+0000","last_clean":"2024-09-14T03:51:33.847346+0000","last_became_active":"2024-09-14T03:44:13.743812+0000","last_became_peered":"2024-09-14T03:44:13.743812+0000","last_unstale":"2024-09-14T03:51:33.847346+0000","last_undegraded":"2024-09-14T03:51:33.847346+0000","last_fullsized":"2024-09-14T03:51:33.847346+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":2322,"log_dups_size":0,"ondisk_log_size":2322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:40:40.022385+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.011055541,"stat_sum":{"num_bytes":29245440,"num_objects":14,"num_object_clones":9,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":1,"num_read":361,"num_read_kb":426063,"num_write":797,"num_write_kb":213368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":93536256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"2.d","version":"58'1","reported_seq":1328,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855536+0000","last_change":"2024-09-14T03:44:13.727000+0000","last_active":"2024-09-14T03:51:31.855536+0000","last_peered":"2024-09-14T03:51:31.855536+0000","last_clean":"2024-09-14T03:51:31.855536+0000","last_became_active":"2024-09-14T03:44:13.726653+0000","last_became_peered":"2024-09-14T03:44:13.726653+0000","last_unstale":"2024-09-14T03:51:31.855536+0000","last_undegraded":"2024-09-14T03:51:31.855536+0000","last_fullsized":"2024-09-14T03:51:31.855536+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:25.998311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1224,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.325923+0000","last_change":"2024-09-14T03:45:20.797918+0000","last_active":"2024-09-14T03:51:34.325923+0000","last_peered":"2024-09-14T03:51:34.325923+0000","last_clean":"2024-09-14T03:51:34.325923+0000","last_became_active":"2024-09-14T03:45:20.797625+0000","last_became_peered":"2024-09-14T03:45:20.797625+0000","last_unstale":"2024-09-14T03:51:34.325923+0000","last_undegraded":"2024-09-14T03:51:34.325923+0000","last_fullsized":"2024-09-14T03:51:34.325923+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:36:59.434260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.b","version":"868'1511","reported_seq":5016,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.847811+0000","last_change":"2024-09-14T03:51:29.876616+0000","last_active":"2024-09-14T03:51:33.847811+0000","last_peered":"2024-09-14T03:51:33.847811+0000","last_clean":"2024-09-14T03:51:33.847811+0000","last_became_active":"2024-09-14T03:45:57.811068+0000","last_became_peered":"2024-09-14T03:45:57.811068+0000","last_unstale":"2024-09-14T03:51:33.847811+0000","last_undegraded":"2024-09-14T03:51:33.847811+0000","last_fullsized":"2024-09-14T03:51:33.847811+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":1511,"log_dups_size":0,"ondisk_log_size":1511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:24.685278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0088557340000000005,"stat_sum":{"num_bytes":38977536,"num_objects":9,"num_object_clones":6,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":245,"num_read_kb":695442,"num_write":634,"num_write_kb":157432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":87482368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"2.c","version":"0'0","reported_seq":1329,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.847257+0000","last_change":"2024-09-14T03:44:47.755095+0000","last_active":"2024-09-14T03:51:33.847257+0000","last_peered":"2024-09-14T03:51:33.847257+0000","last_clean":"2024-09-14T03:51:33.847257+0000","last_became_active":"2024-09-14T03:44:47.754770+0000","last_became_peered":"2024-09-14T03:44:47.754770+0000","last_unstale":"2024-09-14T03:51:33.847257+0000","last_undegraded":"2024-09-14T03:51:33.847257+0000","last_fullsized":"2024-09-14T03:51:33.847257+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:06:07.875846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":1150,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.160311+0000","last_change":"2024-09-14T03:45:54.808288+0000","last_active":"2024-09-14T03:51:32.160311+0000","last_peered":"2024-09-14T03:51:32.160311+0000","last_clean":"2024-09-14T03:51:32.160311+0000","last_became_active":"2024-09-14T03:45:54.808012+0000","last_became_peered":"2024-09-14T03:45:54.808012+0000","last_unstale":"2024-09-14T03:51:32.160311+0000","last_undegraded":"2024-09-14T03:51:32.160311+0000","last_fullsized":"2024-09-14T03:51:32.160311+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:22:32.101743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.4","version":"872'457","reported_seq":3603,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159847+0000","last_change":"2024-09-14T03:51:29.879559+0000","last_active":"2024-09-14T03:51:32.159847+0000","last_peered":"2024-09-14T03:51:32.159847+0000","last_clean":"2024-09-14T03:51:32.159847+0000","last_became_active":"2024-09-14T03:45:54.808844+0000","last_became_peered":"2024-09-14T03:45:54.808844+0000","last_unstale":"2024-09-14T03:51:32.159847+0000","last_undegraded":"2024-09-14T03:51:32.159847+0000","last_fullsized":"2024-09-14T03:51:32.159847+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":457,"log_dups_size":0,"ondisk_log_size":457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:27.723641+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.0071342230000000003,"stat_sum":{"num_bytes":29450240,"num_objects":3,"num_object_clones":2,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":77,"num_read_kb":220061,"num_write":162,"num_write_kb":70504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":4669440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1c1","length":"1"},{"start":"1c5","length":"1"},{"start":"1d7","length":"3"}]},{"pgid":"4.5","version":"0'0","reported_seq":1190,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855198+0000","last_change":"2024-09-14T03:45:54.811298+0000","last_active":"2024-09-14T03:51:31.855198+0000","last_peered":"2024-09-14T03:51:31.855198+0000","last_clean":"2024-09-14T03:51:31.855198+0000","last_became_active":"2024-09-14T03:45:54.811103+0000","last_became_peered":"2024-09-14T03:45:54.811103+0000","last_unstale":"2024-09-14T03:51:31.855198+0000","last_undegraded":"2024-09-14T03:51:31.855198+0000","last_fullsized":"2024-09-14T03:51:31.855198+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:35.359283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030274300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"283'168","reported_seq":1508,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159063+0000","last_change":"2024-09-14T03:45:54.814571+0000","last_active":"2024-09-14T03:51:32.159063+0000","last_peered":"2024-09-14T03:51:32.159063+0000","last_clean":"2024-09-14T03:51:32.159063+0000","last_became_active":"2024-09-14T03:45:54.814134+0000","last_became_peered":"2024-09-14T03:45:54.814134+0000","last_unstale":"2024-09-14T03:51:32.159063+0000","last_undegraded":"2024-09-14T03:51:32.159063+0000","last_fullsized":"2024-09-14T03:51:32.159063+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:31:39.719769+0000","last_clean_scrub_stamp":"2024-09-14T03:31:39.719769+0000","objects_scrubbed":0,"log_size":168,"log_dups_size":0,"ondisk_log_size":168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:09:41.266518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":912,"num_read_kb":1769,"num_write":285,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1237,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.325254+0000","last_change":"2024-09-14T03:45:54.804549+0000","last_active":"2024-09-14T03:51:34.325254+0000","last_peered":"2024-09-14T03:51:34.325254+0000","last_clean":"2024-09-14T03:51:34.325254+0000","last_became_active":"2024-09-14T03:45:54.804262+0000","last_became_peered":"2024-09-14T03:45:54.804262+0000","last_unstale":"2024-09-14T03:51:34.325254+0000","last_undegraded":"2024-09-14T03:51:34.325254+0000","last_fullsized":"2024-09-14T03:51:34.325254+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:29.810427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1273,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861511+0000","last_change":"2024-09-14T03:45:20.805638+0000","last_active":"2024-09-14T03:51:31.861511+0000","last_peered":"2024-09-14T03:51:31.861511+0000","last_clean":"2024-09-14T03:51:31.861511+0000","last_became_active":"2024-09-14T03:45:20.803664+0000","last_became_peered":"2024-09-14T03:45:20.803664+0000","last_unstale":"2024-09-14T03:51:31.861511+0000","last_undegraded":"2024-09-14T03:51:31.861511+0000","last_fullsized":"2024-09-14T03:51:31.861511+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:09:33.593952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.7","version":"868'603","reported_seq":3874,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.327349+0000","last_change":"2024-09-14T03:51:29.876721+0000","last_active":"2024-09-14T03:51:34.327349+0000","last_peered":"2024-09-14T03:51:34.327349+0000","last_clean":"2024-09-14T03:51:34.327349+0000","last_became_active":"2024-09-14T03:44:47.749756+0000","last_became_peered":"2024-09-14T03:44:47.749756+0000","last_unstale":"2024-09-14T03:51:34.327349+0000","last_undegraded":"2024-09-14T03:51:34.327349+0000","last_fullsized":"2024-09-14T03:51:34.327349+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":603,"log_dups_size":0,"ondisk_log_size":603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:49:59.007024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.022817421000000001,"stat_sum":{"num_bytes":8560640,"num_objects":4,"num_object_clones":3,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":64,"num_read_kb":100181,"num_write":276,"num_write_kb":53192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":4317184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"4.6","version":"0'0","reported_seq":1355,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026683+0000","last_change":"2024-09-14T03:43:06.752183+0000","last_active":"2024-09-14T03:51:34.026683+0000","last_peered":"2024-09-14T03:51:34.026683+0000","last_clean":"2024-09-14T03:51:34.026683+0000","last_became_active":"2024-09-14T03:43:06.751899+0000","last_became_peered":"2024-09-14T03:43:06.751899+0000","last_unstale":"2024-09-14T03:51:34.026683+0000","last_undegraded":"2024-09-14T03:51:34.026683+0000","last_fullsized":"2024-09-14T03:51:34.026683+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:27.316549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1158,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.158999+0000","last_change":"2024-09-14T03:45:54.807878+0000","last_active":"2024-09-14T03:51:32.158999+0000","last_peered":"2024-09-14T03:51:32.158999+0000","last_clean":"2024-09-14T03:51:32.158999+0000","last_became_active":"2024-09-14T03:45:54.807236+0000","last_became_peered":"2024-09-14T03:45:54.807236+0000","last_unstale":"2024-09-14T03:51:32.158999+0000","last_undegraded":"2024-09-14T03:51:32.158999+0000","last_fullsized":"2024-09-14T03:51:32.158999+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:13.066293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1386,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026227+0000","last_change":"2024-09-14T03:44:13.730871+0000","last_active":"2024-09-14T03:51:34.026227+0000","last_peered":"2024-09-14T03:51:34.026227+0000","last_clean":"2024-09-14T03:51:34.026227+0000","last_became_active":"2024-09-14T03:44:13.730593+0000","last_became_peered":"2024-09-14T03:44:13.730593+0000","last_unstale":"2024-09-14T03:51:34.026227+0000","last_undegraded":"2024-09-14T03:51:34.026227+0000","last_fullsized":"2024-09-14T03:51:34.026227+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:22:45.210307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":3404,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.847976+0000","last_change":"2024-09-14T03:51:29.867998+0000","last_active":"2024-09-14T03:51:33.847976+0000","last_peered":"2024-09-14T03:51:33.847976+0000","last_clean":"2024-09-14T03:51:33.847976+0000","last_became_active":"2024-09-14T03:44:47.754616+0000","last_became_peered":"2024-09-14T03:44:47.754616+0000","last_unstale":"2024-09-14T03:51:33.847976+0000","last_undegraded":"2024-09-14T03:51:33.847976+0000","last_fullsized":"2024-09-14T03:51:33.847976+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:22:04.608538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054668000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"4.7","version":"0'0","reported_seq":1346,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855047+0000","last_change":"2024-09-14T03:44:47.745505+0000","last_active":"2024-09-14T03:51:31.855047+0000","last_peered":"2024-09-14T03:51:31.855047+0000","last_clean":"2024-09-14T03:51:31.855047+0000","last_became_active":"2024-09-14T03:44:47.745283+0000","last_became_peered":"2024-09-14T03:44:47.745283+0000","last_unstale":"2024-09-14T03:51:31.855047+0000","last_undegraded":"2024-09-14T03:51:31.855047+0000","last_fullsized":"2024-09-14T03:51:31.855047+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:24.359942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027249499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1349,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.846842+0000","last_change":"2024-09-14T03:46:44.763493+0000","last_active":"2024-09-14T03:51:33.846842+0000","last_peered":"2024-09-14T03:51:33.846842+0000","last_clean":"2024-09-14T03:51:33.846842+0000","last_became_active":"2024-09-14T03:46:44.763099+0000","last_became_peered":"2024-09-14T03:46:44.763099+0000","last_unstale":"2024-09-14T03:51:33.846842+0000","last_undegraded":"2024-09-14T03:51:33.846842+0000","last_fullsized":"2024-09-14T03:51:33.846842+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:13:11.384338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1360,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.854737+0000","last_change":"2024-09-14T03:45:20.807508+0000","last_active":"2024-09-14T03:51:31.854737+0000","last_peered":"2024-09-14T03:51:31.854737+0000","last_clean":"2024-09-14T03:51:31.854737+0000","last_became_active":"2024-09-14T03:45:20.807188+0000","last_became_peered":"2024-09-14T03:45:20.807188+0000","last_unstale":"2024-09-14T03:51:31.854737+0000","last_undegraded":"2024-09-14T03:51:31.854737+0000","last_fullsized":"2024-09-14T03:51:31.854737+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:36.775198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"868'1280","reported_seq":4793,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026779+0000","last_change":"2024-09-14T03:51:29.854216+0000","last_active":"2024-09-14T03:51:34.026779+0000","last_peered":"2024-09-14T03:51:34.026779+0000","last_clean":"2024-09-14T03:51:34.026779+0000","last_became_active":"2024-09-14T03:44:13.735888+0000","last_became_peered":"2024-09-14T03:44:13.735888+0000","last_unstale":"2024-09-14T03:51:34.026779+0000","last_undegraded":"2024-09-14T03:51:34.026779+0000","last_fullsized":"2024-09-14T03:51:34.026779+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":1280,"log_dups_size":0,"ondisk_log_size":1280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:50.946344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068819499999999995,"stat_sum":{"num_bytes":33062912,"num_objects":9,"num_object_clones":6,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":257,"num_read_kb":283718,"num_write":433,"num_write_kb":116600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":62185472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"4.4","version":"0'0","reported_seq":1330,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.856049+0000","last_change":"2024-09-14T03:43:39.739302+0000","last_active":"2024-09-14T03:51:31.856049+0000","last_peered":"2024-09-14T03:51:31.856049+0000","last_clean":"2024-09-14T03:51:31.856049+0000","last_became_active":"2024-09-14T03:43:39.739017+0000","last_became_peered":"2024-09-14T03:43:39.739017+0000","last_unstale":"2024-09-14T03:51:31.856049+0000","last_undegraded":"2024-09-14T03:51:31.856049+0000","last_fullsized":"2024-09-14T03:51:31.856049+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:35:38.916582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1206,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.326003+0000","last_change":"2024-09-14T03:45:20.802894+0000","last_active":"2024-09-14T03:51:34.326003+0000","last_peered":"2024-09-14T03:51:34.326003+0000","last_clean":"2024-09-14T03:51:34.326003+0000","last_became_active":"2024-09-14T03:45:20.802570+0000","last_became_peered":"2024-09-14T03:45:20.802570+0000","last_unstale":"2024-09-14T03:51:34.326003+0000","last_undegraded":"2024-09-14T03:51:34.326003+0000","last_fullsized":"2024-09-14T03:51:34.326003+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:04:19.829620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1237,"reported_epoch":873,"state":"active+clean","last_fresh":"2024-09-14T03:51:30.852688+0000","last_change":"2024-09-14T03:45:20.810360+0000","last_active":"2024-09-14T03:51:30.852688+0000","last_peered":"2024-09-14T03:51:30.852688+0000","last_clean":"2024-09-14T03:51:30.852688+0000","last_became_active":"2024-09-14T03:45:20.810088+0000","last_became_peered":"2024-09-14T03:45:20.810088+0000","last_unstale":"2024-09-14T03:51:30.852688+0000","last_undegraded":"2024-09-14T03:51:30.852688+0000","last_fullsized":"2024-09-14T03:51:30.852688+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:37:37.573371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.3","version":"873'420","reported_seq":3949,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026769+0000","last_change":"2024-09-14T03:51:29.853890+0000","last_active":"2024-09-14T03:51:34.026769+0000","last_peered":"2024-09-14T03:51:34.026769+0000","last_clean":"2024-09-14T03:51:34.026769+0000","last_became_active":"2024-09-14T03:45:20.807094+0000","last_became_peered":"2024-09-14T03:45:20.807094+0000","last_unstale":"2024-09-14T03:51:34.026769+0000","last_undegraded":"2024-09-14T03:51:34.026769+0000","last_fullsized":"2024-09-14T03:51:34.026769+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":420,"log_dups_size":0,"ondisk_log_size":420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:04.467905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074506099999999999,"stat_sum":{"num_bytes":1744896,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":1,"num_read":64,"num_read_kb":80911,"num_write":208,"num_write_kb":42784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"4.2","version":"67'2","reported_seq":1332,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855653+0000","last_change":"2024-09-14T03:44:47.755235+0000","last_active":"2024-09-14T03:51:31.855653+0000","last_peered":"2024-09-14T03:51:31.855653+0000","last_clean":"2024-09-14T03:51:31.855653+0000","last_became_active":"2024-09-14T03:44:47.754661+0000","last_became_peered":"2024-09-14T03:44:47.754661+0000","last_unstale":"2024-09-14T03:51:31.855653+0000","last_undegraded":"2024-09-14T03:51:31.855653+0000","last_fullsized":"2024-09-14T03:51:31.855653+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:20:48.981284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1351,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855946+0000","last_change":"2024-09-14T03:45:54.804715+0000","last_active":"2024-09-14T03:51:31.855946+0000","last_peered":"2024-09-14T03:51:31.855946+0000","last_clean":"2024-09-14T03:51:31.855946+0000","last_became_active":"2024-09-14T03:45:54.804380+0000","last_became_peered":"2024-09-14T03:45:54.804380+0000","last_unstale":"2024-09-14T03:51:31.855946+0000","last_undegraded":"2024-09-14T03:51:31.855946+0000","last_fullsized":"2024-09-14T03:51:31.855946+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:52.997838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1230,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.326991+0000","last_change":"2024-09-14T03:44:47.748246+0000","last_active":"2024-09-14T03:51:34.326991+0000","last_peered":"2024-09-14T03:51:34.326991+0000","last_clean":"2024-09-14T03:51:34.326991+0000","last_became_active":"2024-09-14T03:44:47.746928+0000","last_became_peered":"2024-09-14T03:44:47.746928+0000","last_unstale":"2024-09-14T03:51:34.326991+0000","last_undegraded":"2024-09-14T03:51:34.326991+0000","last_fullsized":"2024-09-14T03:51:34.326991+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:33:10.836876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.2","version":"868'1169","reported_seq":4341,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855085+0000","last_change":"2024-09-14T03:51:29.857344+0000","last_active":"2024-09-14T03:51:31.855085+0000","last_peered":"2024-09-14T03:51:31.855085+0000","last_clean":"2024-09-14T03:51:31.855085+0000","last_became_active":"2024-09-14T03:45:20.811086+0000","last_became_peered":"2024-09-14T03:45:20.811086+0000","last_unstale":"2024-09-14T03:51:31.855085+0000","last_undegraded":"2024-09-14T03:51:31.855085+0000","last_fullsized":"2024-09-14T03:51:31.855085+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":1169,"log_dups_size":0,"ondisk_log_size":1169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:13.139908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000679136,"stat_sum":{"num_bytes":5169152,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":237,"num_read_kb":290917,"num_write":346,"num_write_kb":122536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":28327936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"1c1","length":"1"},{"start":"1c5","length":"1"},{"start":"1d7","length":"3"}]},{"pgid":"4.3","version":"65'1","reported_seq":1356,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026678+0000","last_change":"2024-09-14T03:45:54.806304+0000","last_active":"2024-09-14T03:51:34.026678+0000","last_peered":"2024-09-14T03:51:34.026678+0000","last_clean":"2024-09-14T03:51:34.026678+0000","last_became_active":"2024-09-14T03:45:54.805662+0000","last_became_peered":"2024-09-14T03:45:54.805662+0000","last_unstale":"2024-09-14T03:51:34.026678+0000","last_undegraded":"2024-09-14T03:51:34.026678+0000","last_fullsized":"2024-09-14T03:51:34.026678+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:50:54.074166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":650,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159904+0000","last_change":"2024-09-14T03:45:55.797061+0000","last_active":"2024-09-14T03:51:32.159904+0000","last_peered":"2024-09-14T03:51:32.159904+0000","last_clean":"2024-09-14T03:51:32.159904+0000","last_became_active":"2024-09-14T03:45:55.796743+0000","last_became_peered":"2024-09-14T03:45:55.796743+0000","last_unstale":"2024-09-14T03:51:32.159904+0000","last_undegraded":"2024-09-14T03:51:32.159904+0000","last_fullsized":"2024-09-14T03:51:32.159904+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:29:01.209584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1360,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855824+0000","last_change":"2024-09-14T03:44:47.744907+0000","last_active":"2024-09-14T03:51:31.855824+0000","last_peered":"2024-09-14T03:51:31.855824+0000","last_clean":"2024-09-14T03:51:31.855824+0000","last_became_active":"2024-09-14T03:44:47.744267+0000","last_became_peered":"2024-09-14T03:44:47.744267+0000","last_unstale":"2024-09-14T03:51:31.855824+0000","last_undegraded":"2024-09-14T03:51:31.855824+0000","last_fullsized":"2024-09-14T03:51:31.855824+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:54:32.360947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.1","version":"868'2115","reported_seq":5272,"reported_epoch":873,"state":"active+clean","last_fresh":"2024-09-14T03:51:30.852663+0000","last_change":"2024-09-14T03:51:29.848772+0000","last_active":"2024-09-14T03:51:30.852663+0000","last_peered":"2024-09-14T03:51:30.852663+0000","last_clean":"2024-09-14T03:51:30.852663+0000","last_became_active":"2024-09-14T03:45:54.811401+0000","last_became_peered":"2024-09-14T03:45:54.811401+0000","last_unstale":"2024-09-14T03:51:30.852663+0000","last_undegraded":"2024-09-14T03:51:30.852663+0000","last_fullsized":"2024-09-14T03:51:30.852663+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":2115,"log_dups_size":0,"ondisk_log_size":2115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:44.908297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000801113,"stat_sum":{"num_bytes":40722432,"num_objects":16,"num_object_clones":11,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":2,"num_read":319,"num_read_kb":397041,"num_write":709,"num_write_kb":178464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":49078272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1c1","length":"1"},{"start":"1c5","length":"1"},{"start":"1d7","length":"3"}]},{"pgid":"4.0","version":"0'0","reported_seq":1300,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.860696+0000","last_change":"2024-09-14T03:45:54.800835+0000","last_active":"2024-09-14T03:51:31.860696+0000","last_peered":"2024-09-14T03:51:31.860696+0000","last_clean":"2024-09-14T03:51:31.860696+0000","last_became_active":"2024-09-14T03:45:54.800551+0000","last_became_peered":"2024-09-14T03:45:54.800551+0000","last_unstale":"2024-09-14T03:51:31.860696+0000","last_undegraded":"2024-09-14T03:51:31.860696+0000","last_fullsized":"2024-09-14T03:51:31.860696+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:29:40.121693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070668399999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"579'3","reported_seq":1330,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855552+0000","last_change":"2024-09-14T03:45:20.808027+0000","last_active":"2024-09-14T03:51:31.855552+0000","last_peered":"2024-09-14T03:51:31.855552+0000","last_clean":"2024-09-14T03:51:31.855552+0000","last_became_active":"2024-09-14T03:45:20.807656+0000","last_became_peered":"2024-09-14T03:45:20.807656+0000","last_unstale":"2024-09-14T03:51:31.855552+0000","last_undegraded":"2024-09-14T03:51:31.855552+0000","last_fullsized":"2024-09-14T03:51:31.855552+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:09:08.609241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":1302,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861769+0000","last_change":"2024-09-14T03:45:54.800748+0000","last_active":"2024-09-14T03:51:31.861769+0000","last_peered":"2024-09-14T03:51:31.861769+0000","last_clean":"2024-09-14T03:51:31.861769+0000","last_became_active":"2024-09-14T03:45:54.800443+0000","last_became_peered":"2024-09-14T03:45:54.800443+0000","last_unstale":"2024-09-14T03:51:31.861769+0000","last_undegraded":"2024-09-14T03:51:31.861769+0000","last_fullsized":"2024-09-14T03:51:31.861769+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:18.504124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"873'1324","reported_seq":4642,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861551+0000","last_change":"2024-09-14T03:51:29.849279+0000","last_active":"2024-09-14T03:51:31.861551+0000","last_peered":"2024-09-14T03:51:31.861551+0000","last_clean":"2024-09-14T03:51:31.861551+0000","last_became_active":"2024-09-14T03:44:13.737279+0000","last_became_peered":"2024-09-14T03:44:13.737279+0000","last_unstale":"2024-09-14T03:51:31.861551+0000","last_undegraded":"2024-09-14T03:51:31.861551+0000","last_fullsized":"2024-09-14T03:51:31.861551+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":1324,"log_dups_size":0,"ondisk_log_size":1324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:51:53.459635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012983059999999999,"stat_sum":{"num_bytes":16441344,"num_objects":9,"num_object_clones":6,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":230,"num_read_kb":260011,"num_write":485,"num_write_kb":74800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":21463040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"1c1","length":"1"},{"start":"1c5","length":"1"},{"start":"1d7","length":"3"}]},{"pgid":"4.1","version":"0'0","reported_seq":1258,"reported_epoch":873,"state":"active+clean","last_fresh":"2024-09-14T03:51:30.852772+0000","last_change":"2024-09-14T03:45:20.813312+0000","last_active":"2024-09-14T03:51:30.852772+0000","last_peered":"2024-09-14T03:51:30.852772+0000","last_clean":"2024-09-14T03:51:30.852772+0000","last_became_active":"2024-09-14T03:45:20.812948+0000","last_became_peered":"2024-09-14T03:45:20.812948+0000","last_unstale":"2024-09-14T03:51:30.852772+0000","last_undegraded":"2024-09-14T03:51:30.852772+0000","last_fullsized":"2024-09-14T03:51:30.852772+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:50.989704+0000","last_clean_scrub_stamp":"2024-09-14T03:34:50.989704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:00:08.585007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":1175,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855154+0000","last_change":"2024-09-14T03:45:54.801330+0000","last_active":"2024-09-14T03:51:31.855154+0000","last_peered":"2024-09-14T03:51:31.855154+0000","last_clean":"2024-09-14T03:51:31.855154+0000","last_became_active":"2024-09-14T03:45:54.801024+0000","last_became_peered":"2024-09-14T03:45:54.801024+0000","last_unstale":"2024-09-14T03:51:31.855154+0000","last_undegraded":"2024-09-14T03:51:31.855154+0000","last_fullsized":"2024-09-14T03:51:31.855154+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:07:41.528985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":9,"num_read_kb":9,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":1386,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026616+0000","last_change":"2024-09-14T03:44:13.728163+0000","last_active":"2024-09-14T03:51:34.026616+0000","last_peered":"2024-09-14T03:51:34.026616+0000","last_clean":"2024-09-14T03:51:34.026616+0000","last_became_active":"2024-09-14T03:44:13.727870+0000","last_became_peered":"2024-09-14T03:44:13.727870+0000","last_unstale":"2024-09-14T03:51:34.026616+0000","last_undegraded":"2024-09-14T03:51:34.026616+0000","last_fullsized":"2024-09-14T03:51:34.026616+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:43:32.359660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.f","version":"872'2866","reported_seq":5958,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.327208+0000","last_change":"2024-09-14T03:51:29.875838+0000","last_active":"2024-09-14T03:51:34.327208+0000","last_peered":"2024-09-14T03:51:34.327208+0000","last_clean":"2024-09-14T03:51:34.327208+0000","last_became_active":"2024-09-14T03:45:20.803136+0000","last_became_peered":"2024-09-14T03:45:20.803136+0000","last_unstale":"2024-09-14T03:51:34.327208+0000","last_undegraded":"2024-09-14T03:51:34.327208+0000","last_fullsized":"2024-09-14T03:51:34.327208+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":2866,"log_dups_size":0,"ondisk_log_size":2866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:34:20.553013+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.022140343,"stat_sum":{"num_bytes":70270976,"num_objects":22,"num_object_clones":15,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":513,"num_read_kb":658581,"num_write":1226,"num_write_kb":278584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":52617216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"2.8","version":"0'0","reported_seq":1158,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159097+0000","last_change":"2024-09-14T03:45:54.808148+0000","last_active":"2024-09-14T03:51:32.159097+0000","last_peered":"2024-09-14T03:51:32.159097+0000","last_clean":"2024-09-14T03:51:32.159097+0000","last_became_active":"2024-09-14T03:45:54.807592+0000","last_became_peered":"2024-09-14T03:45:54.807592+0000","last_unstale":"2024-09-14T03:51:32.159097+0000","last_undegraded":"2024-09-14T03:51:32.159097+0000","last_fullsized":"2024-09-14T03:51:32.159097+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:06.367185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1260,"reported_epoch":873,"state":"active+clean","last_fresh":"2024-09-14T03:51:30.853012+0000","last_change":"2024-09-14T03:45:54.811366+0000","last_active":"2024-09-14T03:51:30.853012+0000","last_peered":"2024-09-14T03:51:30.853012+0000","last_clean":"2024-09-14T03:51:30.853012+0000","last_became_active":"2024-09-14T03:45:54.811063+0000","last_became_peered":"2024-09-14T03:45:54.811063+0000","last_unstale":"2024-09-14T03:51:30.853012+0000","last_undegraded":"2024-09-14T03:51:30.853012+0000","last_fullsized":"2024-09-14T03:51:30.853012+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:36.782644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"872'2444","reported_seq":5843,"reported_epoch":873,"state":"active+clean","last_fresh":"2024-09-14T03:51:30.853356+0000","last_change":"2024-09-14T03:51:29.855260+0000","last_active":"2024-09-14T03:51:30.853356+0000","last_peered":"2024-09-14T03:51:30.853356+0000","last_clean":"2024-09-14T03:51:30.853356+0000","last_became_active":"2024-09-14T03:45:46.749599+0000","last_became_peered":"2024-09-14T03:45:46.749599+0000","last_unstale":"2024-09-14T03:51:30.853356+0000","last_undegraded":"2024-09-14T03:51:30.853356+0000","last_fullsized":"2024-09-14T03:51:30.853356+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":2444,"log_dups_size":0,"ondisk_log_size":2444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:21.041013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0065258470000000004,"stat_sum":{"num_bytes":31809536,"num_objects":15,"num_object_clones":10,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":1,"num_read":409,"num_read_kb":657258,"num_write":893,"num_write_kb":220040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":82313216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1c1","length":"1"},{"start":"1c5","length":"1"},{"start":"1d7","length":"3"}]},{"pgid":"2.9","version":"0'0","reported_seq":1361,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855635+0000","last_change":"2024-09-14T03:45:54.804546+0000","last_active":"2024-09-14T03:51:31.855635+0000","last_peered":"2024-09-14T03:51:31.855635+0000","last_clean":"2024-09-14T03:51:31.855635+0000","last_became_active":"2024-09-14T03:45:54.804212+0000","last_became_peered":"2024-09-14T03:45:54.804212+0000","last_unstale":"2024-09-14T03:51:31.855635+0000","last_undegraded":"2024-09-14T03:51:31.855635+0000","last_fullsized":"2024-09-14T03:51:31.855635+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:24:52.058255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1295,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861362+0000","last_change":"2024-09-14T03:45:54.801769+0000","last_active":"2024-09-14T03:51:31.861362+0000","last_peered":"2024-09-14T03:51:31.861362+0000","last_clean":"2024-09-14T03:51:31.861362+0000","last_became_active":"2024-09-14T03:45:54.801430+0000","last_became_peered":"2024-09-14T03:45:54.801430+0000","last_unstale":"2024-09-14T03:51:31.861362+0000","last_undegraded":"2024-09-14T03:51:31.861362+0000","last_fullsized":"2024-09-14T03:51:31.861362+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:26:50.653214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.d","version":"872'1244","reported_seq":4586,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:33.847646+0000","last_change":"2024-09-14T03:51:29.876038+0000","last_active":"2024-09-14T03:51:33.847646+0000","last_peered":"2024-09-14T03:51:33.847646+0000","last_clean":"2024-09-14T03:51:33.847646+0000","last_became_active":"2024-09-14T03:45:54.802817+0000","last_became_peered":"2024-09-14T03:45:54.802817+0000","last_unstale":"2024-09-14T03:51:33.847646+0000","last_undegraded":"2024-09-14T03:51:33.847646+0000","last_fullsized":"2024-09-14T03:51:33.847646+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":1244,"log_dups_size":0,"ondisk_log_size":1244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:37:18.288839+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.0085178890000000007,"stat_sum":{"num_bytes":17719296,"num_objects":7,"num_object_clones":4,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":178,"num_read_kb":293925,"num_write":465,"num_write_kb":124656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":37576704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"2"}]},{"pgid":"2.a","version":"0'0","reported_seq":1361,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855329+0000","last_change":"2024-09-14T03:45:54.797372+0000","last_active":"2024-09-14T03:51:31.855329+0000","last_peered":"2024-09-14T03:51:31.855329+0000","last_clean":"2024-09-14T03:51:31.855329+0000","last_became_active":"2024-09-14T03:45:54.797050+0000","last_became_peered":"2024-09-14T03:45:54.797050+0000","last_unstale":"2024-09-14T03:51:31.855329+0000","last_undegraded":"2024-09-14T03:51:31.855329+0000","last_fullsized":"2024-09-14T03:51:31.855329+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:51:15.853615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":1302,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861110+0000","last_change":"2024-09-14T03:44:13.734515+0000","last_active":"2024-09-14T03:51:31.861110+0000","last_peered":"2024-09-14T03:51:31.861110+0000","last_clean":"2024-09-14T03:51:31.861110+0000","last_became_active":"2024-09-14T03:44:13.734217+0000","last_became_peered":"2024-09-14T03:44:13.734217+0000","last_unstale":"2024-09-14T03:51:31.861110+0000","last_undegraded":"2024-09-14T03:51:31.861110+0000","last_fullsized":"2024-09-14T03:51:31.861110+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:50.831462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.c","version":"872'2186","reported_seq":5530,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.856063+0000","last_change":"2024-09-14T03:51:29.855268+0000","last_active":"2024-09-14T03:51:31.856063+0000","last_peered":"2024-09-14T03:51:31.856063+0000","last_clean":"2024-09-14T03:51:31.856063+0000","last_became_active":"2024-09-14T03:44:13.729252+0000","last_became_peered":"2024-09-14T03:44:13.729252+0000","last_unstale":"2024-09-14T03:51:31.856063+0000","last_undegraded":"2024-09-14T03:51:31.856063+0000","last_fullsized":"2024-09-14T03:51:31.856063+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:37:35.999294+0000","last_clean_scrub_stamp":"2024-09-14T03:37:35.999294+0000","objects_scrubbed":0,"log_size":2186,"log_dups_size":0,"ondisk_log_size":2186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:27:18.624021+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.0058310949999999997,"stat_sum":{"num_bytes":38043648,"num_objects":15,"num_object_clones":10,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":1,"num_read":372,"num_read_kb":991451,"num_write":868,"num_write_kb":180288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":44802048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"1c1","length":"1"},{"start":"1c5","length":"1"},{"start":"1d7","length":"3"}]},{"pgid":"2.b","version":"576'4","reported_seq":1169,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159371+0000","last_change":"2024-09-14T03:45:54.814296+0000","last_active":"2024-09-14T03:51:32.159371+0000","last_peered":"2024-09-14T03:51:32.159371+0000","last_clean":"2024-09-14T03:51:32.159371+0000","last_became_active":"2024-09-14T03:45:54.811431+0000","last_became_peered":"2024-09-14T03:45:54.811431+0000","last_unstale":"2024-09-14T03:51:32.159371+0000","last_undegraded":"2024-09-14T03:51:32.159371+0000","last_fullsized":"2024-09-14T03:51:32.159371+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:59.963051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1197,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.855202+0000","last_change":"2024-09-14T03:45:20.806912+0000","last_active":"2024-09-14T03:51:31.855202+0000","last_peered":"2024-09-14T03:51:31.855202+0000","last_clean":"2024-09-14T03:51:31.855202+0000","last_became_active":"2024-09-14T03:45:20.806658+0000","last_became_peered":"2024-09-14T03:45:20.806658+0000","last_unstale":"2024-09-14T03:51:31.855202+0000","last_undegraded":"2024-09-14T03:51:31.855202+0000","last_fullsized":"2024-09-14T03:51:31.855202+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:36.793980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":1215,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.326763+0000","last_change":"2024-09-14T03:45:20.797497+0000","last_active":"2024-09-14T03:51:34.326763+0000","last_peered":"2024-09-14T03:51:34.326763+0000","last_clean":"2024-09-14T03:51:34.326763+0000","last_became_active":"2024-09-14T03:45:20.797219+0000","last_became_peered":"2024-09-14T03:45:20.797219+0000","last_unstale":"2024-09-14T03:51:34.326763+0000","last_undegraded":"2024-09-14T03:51:34.326763+0000","last_fullsized":"2024-09-14T03:51:34.326763+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:41:36.186368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":1168,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.159559+0000","last_change":"2024-09-14T03:45:54.808225+0000","last_active":"2024-09-14T03:51:32.159559+0000","last_peered":"2024-09-14T03:51:32.159559+0000","last_clean":"2024-09-14T03:51:32.159559+0000","last_became_active":"2024-09-14T03:45:54.807824+0000","last_became_peered":"2024-09-14T03:45:54.807824+0000","last_unstale":"2024-09-14T03:51:32.159559+0000","last_undegraded":"2024-09-14T03:51:32.159559+0000","last_fullsized":"2024-09-14T03:51:32.159559+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:58:22.642844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":9,"num_read_kb":9,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":1221,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.326025+0000","last_change":"2024-09-14T03:44:47.747178+0000","last_active":"2024-09-14T03:51:34.326025+0000","last_peered":"2024-09-14T03:51:34.326025+0000","last_clean":"2024-09-14T03:51:34.326025+0000","last_became_active":"2024-09-14T03:44:47.746253+0000","last_became_peered":"2024-09-14T03:44:47.746253+0000","last_unstale":"2024-09-14T03:51:34.326025+0000","last_undegraded":"2024-09-14T03:51:34.326025+0000","last_fullsized":"2024-09-14T03:51:34.326025+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:01.069988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":1158,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:32.160196+0000","last_change":"2024-09-14T03:45:54.811739+0000","last_active":"2024-09-14T03:51:32.160196+0000","last_peered":"2024-09-14T03:51:32.160196+0000","last_clean":"2024-09-14T03:51:32.160196+0000","last_became_active":"2024-09-14T03:45:54.811415+0000","last_became_peered":"2024-09-14T03:45:54.811415+0000","last_unstale":"2024-09-14T03:51:32.160196+0000","last_undegraded":"2024-09-14T03:51:32.160196+0000","last_fullsized":"2024-09-14T03:51:32.160196+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:08:30.854590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":9,"num_read_kb":9,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":1386,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.026690+0000","last_change":"2024-09-14T03:44:47.743019+0000","last_active":"2024-09-14T03:51:34.026690+0000","last_peered":"2024-09-14T03:51:34.026690+0000","last_clean":"2024-09-14T03:51:34.026690+0000","last_became_active":"2024-09-14T03:44:47.742570+0000","last_became_peered":"2024-09-14T03:44:47.742570+0000","last_unstale":"2024-09-14T03:51:34.026690+0000","last_undegraded":"2024-09-14T03:51:34.026690+0000","last_fullsized":"2024-09-14T03:51:34.026690+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:14.512057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"576'3","reported_seq":1308,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861950+0000","last_change":"2024-09-14T03:44:47.741204+0000","last_active":"2024-09-14T03:51:31.861950+0000","last_peered":"2024-09-14T03:51:31.861950+0000","last_clean":"2024-09-14T03:51:31.861950+0000","last_became_active":"2024-09-14T03:44:47.740902+0000","last_became_peered":"2024-09-14T03:44:47.740902+0000","last_unstale":"2024-09-14T03:51:31.861950+0000","last_undegraded":"2024-09-14T03:51:31.861950+0000","last_fullsized":"2024-09-14T03:51:31.861950+0000","mapping_epoch":486,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:13.197948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":1284,"reported_epoch":874,"state":"active+clean","last_fresh":"2024-09-14T03:51:31.861871+0000","last_change":"2024-09-14T03:45:20.804648+0000","last_active":"2024-09-14T03:51:31.861871+0000","last_peered":"2024-09-14T03:51:31.861871+0000","last_clean":"2024-09-14T03:51:31.861871+0000","last_became_active":"2024-09-14T03:45:20.804056+0000","last_became_peered":"2024-09-14T03:45:20.804056+0000","last_unstale":"2024-09-14T03:51:31.861871+0000","last_undegraded":"2024-09-14T03:51:31.861871+0000","last_fullsized":"2024-09-14T03:51:31.861871+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:34:01.486715+0000","last_clean_scrub_stamp":"2024-09-14T03:34:01.486715+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:10.811855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"576'6","reported_seq":1401,"reported_epoch":875,"state":"active+clean","last_fresh":"2024-09-14T03:51:34.027007+0000","last_change":"2024-09-14T03:45:54.810215+0000","last_active":"2024-09-14T03:51:34.027007+0000","last_peered":"2024-09-14T03:51:34.027007+0000","last_clean":"2024-09-14T03:51:34.027007+0000","last_became_active":"2024-09-14T03:45:54.809901+0000","last_became_peered":"2024-09-14T03:45:54.809901+0000","last_unstale":"2024-09-14T03:51:34.027007+0000","last_undegraded":"2024-09-14T03:51:34.027007+0000","last_fullsized":"2024-09-14T03:51:34.027007+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:33:58.386582+0000","last_clean_scrub_stamp":"2024-09-14T03:33:58.386582+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:19.656678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":8,"num_read_kb":8,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":390430720,"num_objects":140,"num_object_clones":92,"num_object_copies":420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":140,"num_whiteouts":6,"num_read":3642,"num_read_kb":5880788,"num_write":8330,"num_write_kb":2046808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":318,"num_bytes_recovered":605691904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":550477824,"data_stored":894828544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22041,"ondisk_log_size":22041,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":6602,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":120,"num_read_kb":128,"num_write":69,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":19806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45852,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":912,"num_read_kb":1769,"num_write":285,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":168,"ondisk_log_size":168,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":6,"up_from":518,"seq":2224793059404,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":235712,"kb_used_data":56116,"kb_used_omap":35,"kb_used_meta":179548,"kb_avail":93509440,"statfs":{"total":95995035648,"available":95753666560,"internally_reserved":0,"allocated":57462784,"data_stored":68800753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36545,"internal_metadata":183857471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":454,"seq":1949915152473,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":678892,"kb_used_data":126176,"kb_used_omap":66,"kb_used_meta":552637,"kb_avail":93066260,"statfs":{"total":95995035648,"available":95299850240,"internally_reserved":0,"allocated":129204224,"data_stored":195372380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68143,"internal_metadata":565900753},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":420,"seq":1803886264416,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":338312,"kb_used_data":64948,"kb_used_omap":30,"kb_used_meta":273313,"kb_avail":93406840,"statfs":{"total":95995035648,"available":95648604160,"internally_reserved":0,"allocated":66506752,"data_stored":95113025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31112,"internal_metadata":279873144},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":389,"seq":1670742278247,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":414060,"kb_used_data":70944,"kb_used_omap":29,"kb_used_meta":343074,"kb_avail":93331092,"statfs":{"total":95995035648,"available":95571038208,"internally_reserved":0,"allocated":72646656,"data_stored":104168644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30662,"internal_metadata":351307834},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":358,"seq":1537598292077,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416756,"kb_used_data":61732,"kb_used_omap":37,"kb_used_meta":354970,"kb_avail":93328396,"statfs":{"total":95995035648,"available":95568277504,"internally_reserved":0,"allocated":63213568,"data_stored":102774996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37919,"internal_metadata":363490273},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":552,"seq":2370821947461,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409964,"kb_used_data":81560,"kb_used_omap":33,"kb_used_meta":328350,"kb_avail":93335188,"statfs":{"total":95995035648,"available":95575232512,"internally_reserved":0,"allocated":83517440,"data_stored":120913033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34036,"internal_metadata":336231180},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":486,"seq":2087354105939,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":534940,"kb_used_data":91600,"kb_used_omap":49,"kb_used_meta":443278,"kb_avail":93210212,"statfs":{"total":95995035648,"available":95447257088,"internally_reserved":0,"allocated":93798400,"data_stored":146330569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51052,"internal_metadata":453916820},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":324,"seq":1391569404020,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590428,"kb_used_data":63752,"kb_used_omap":45,"kb_used_meta":526610,"kb_avail":93154724,"statfs":{"total":95995035648,"available":95390437376,"internally_reserved":0,"allocated":65282048,"data_stored":109407501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46664,"internal_metadata":539249080},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5944,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4559,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4994,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6514,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6456,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6034,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":5816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5344,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6007,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":54767616,"data_stored":103034880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":53297152,"data_stored":96993280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":62726144,"data_stored":98377728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":56578048,"data_stored":89325568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":119296000,"data_stored":189599744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":83853312,"data_stored":140533760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":46948352,"data_stored":62423040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":73011200,"data_stored":114540544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:51:37.131 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-14T03:51:37.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-14T03:51:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:36 smithi195 ceph-mon[82388]: pgmap v951: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 765 KiB/s wr, 6 op/s 2024-09-14T03:51:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:36 smithi195 ceph-mon[82388]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:51:37.376 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:38.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:38 smithi110 ceph-mon[88262]: pgmap v952: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 340 KiB/s wr, 3 op/s 2024-09-14T03:51:38.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:38 smithi110 ceph-mon[88262]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T03:51:38.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:38 smithi110 ceph-mon[86423]: pgmap v952: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 340 KiB/s wr, 3 op/s 2024-09-14T03:51:38.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:38 smithi110 ceph-mon[86423]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T03:51:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:38 smithi195 ceph-mon[82388]: pgmap v952: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 340 KiB/s wr, 3 op/s 2024-09-14T03:51:39.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:38 smithi195 ceph-mon[82388]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T03:51:39.879 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_0' removed 2024-09-14T03:51:40.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:39 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/861494008' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:51:40.188 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:39 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/861494008' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:51:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:39 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/861494008' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:51:40.500 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-14T03:51:40.500 INFO:teuthology.task.print:**** done end ec-rados-default.yaml 2024-09-14T03:51:40.500 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-14T03:51:40.500 INFO:teuthology.task.print:**** done start rados_api.yaml 2024-09-14T03:51:40.500 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-14T03:51:40.501 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-14T03:51:40.502 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T03:51:40.502 DEBUG:teuthology.orchestra.run.smithi110:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:51:40.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:51:40.519 INFO:teuthology.orchestra.run.smithi110.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-14T03:51:40.520 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:51:40.576 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-14T03:51:40.577 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-14T03:51:40.633 INFO:tasks.workunit:timeout=3h 2024-09-14T03:51:40.633 INFO:tasks.workunit:cleanup=True 2024-09-14T03:51:40.633 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-14T03:51:40.690 INFO:tasks.workunit.client.0.smithi110.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-14T03:51:40.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:51:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:51:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:40 smithi110 ceph-mon[88262]: pgmap v954: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 298 KiB/s wr, 3 op/s 2024-09-14T03:51:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:40 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/861494008' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:51:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:40 smithi110 ceph-mon[88262]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T03:51:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:40 smithi110 ceph-mon[86423]: pgmap v954: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 298 KiB/s wr, 3 op/s 2024-09-14T03:51:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:40 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/861494008' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:51:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:40 smithi110 ceph-mon[86423]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T03:51:41.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:40 smithi195 ceph-mon[82388]: pgmap v954: 121 pgs: 121 active+clean; 373 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 298 KiB/s wr, 3 op/s 2024-09-14T03:51:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:40 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/861494008' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:51:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:40 smithi195 ceph-mon[82388]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T03:51:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:51:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:51:41] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-14T03:51:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:42 smithi110 ceph-mon[86423]: pgmap v956: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:42 smithi110 ceph-mon[88262]: pgmap v956: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:43.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:42 smithi195 ceph-mon[82388]: pgmap v956: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:43.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:44 smithi110 ceph-mon[88262]: pgmap v957: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:44 smithi110 ceph-mon[86423]: pgmap v957: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:44 smithi195 ceph-mon[82388]: pgmap v957: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:46.869 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-14T03:51:46.925 INFO:tasks.workunit.client.0.smithi110.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T03:51:46.927 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T03:51:46.928 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T03:51:46.976 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T03:51:47.025 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T03:51:47.064 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T03:51:47.065 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T03:51:47.065 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T03:51:47.103 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T03:51:47.106 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-14T03:51:47.106 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-14T03:51:47.174 INFO:tasks.workunit:Running workunits matching cls on client.0... 2024-09-14T03:51:47.175 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2024-09-14T03:51:47.175 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2024-09-14T03:51:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:46 smithi110 ceph-mon[86423]: pgmap v958: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:46 smithi110 ceph-mon[88262]: pgmap v958: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:47.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:46 smithi195 ceph-mon[82388]: pgmap v958: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:47.222 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:51:47.222 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 22 tests from 1 test suite. 2024-09-14T03:51:47.222 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:51:47.222 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 22 tests from TestCls2PCQueue 2024-09-14T03:51:47.222 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2024-09-14T03:51:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:48 smithi110 ceph-mon[86423]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:48 smithi110 ceph-mon[86423]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T03:51:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:48 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/76717940' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:48 smithi110 ceph-mon[88262]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:49.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:48 smithi110 ceph-mon[88262]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T03:51:49.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:48 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/76717940' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:49.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:48 smithi195 ceph-mon[82388]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:48 smithi195 ceph-mon[82388]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T03:51:49.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:48 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/76717940' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:49.912 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2691 ms) 2024-09-14T03:51:49.913 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2024-09-14T03:51:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:49 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/76717940' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:49 smithi110 ceph-mon[88262]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T03:51:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:49 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/76717940' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:49 smithi110 ceph-mon[86423]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T03:51:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:49 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/76717940' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:49 smithi195 ceph-mon[82388]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T03:51:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:51:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:51:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:50 smithi110 ceph-mon[86423]: pgmap v962: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:50 smithi110 ceph-mon[86423]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T03:51:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:50 smithi110 ceph-mon[88262]: pgmap v962: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:50 smithi110 ceph-mon[88262]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T03:51:51.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:50 smithi195 ceph-mon[82388]: pgmap v962: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:50 smithi195 ceph-mon[82388]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T03:51:51.920 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:51:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:51:51] "GET /metrics HTTP/1.1" 200 33955 "" "Prometheus/2.51.0" 2024-09-14T03:51:52.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:51 smithi110 ceph-mon[88262]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T03:51:52.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:51 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/242052871' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:52.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:51 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:52.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:51 smithi110 ceph-mon[86423]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T03:51:52.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:51 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/242052871' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:52.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:51 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:51 smithi195 ceph-mon[82388]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T03:51:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:51 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/242052871' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:51 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:52.931 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3019 ms) 2024-09-14T03:51:52.931 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.Reserve 2024-09-14T03:51:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:52 smithi110 ceph-mon[88262]: pgmap v965: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:52 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:52 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:53.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:52 smithi110 ceph-mon[88262]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T03:51:53.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:52 smithi110 ceph-mon[86423]: pgmap v965: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:52 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:52 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:53.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:52 smithi110 ceph-mon[86423]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T03:51:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:52 smithi195 ceph-mon[82388]: pgmap v965: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:52 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:52 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:52 smithi195 ceph-mon[82388]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T03:51:54.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:53 smithi110 ceph-mon[88262]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T03:51:54.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:53 smithi110 ceph-mon[86423]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T03:51:54.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:53 smithi195 ceph-mon[82388]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T03:51:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[86423]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:51:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[86423]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/503526284' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[88262]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[88262]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/503526284' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:55.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:54 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:55.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:54 smithi195 ceph-mon[82388]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:54 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:51:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:54 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:51:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:54 smithi195 ceph-mon[82388]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T03:51:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:54 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/503526284' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:55.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:54 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:55.951 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.Reserve (3020 ms) 2024-09-14T03:51:55.951 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2024-09-14T03:51:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:55 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:55 smithi110 ceph-mon[88262]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T03:51:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:55 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:56.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:55 smithi110 ceph-mon[86423]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T03:51:56.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:55 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:56.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:55 smithi195 ceph-mon[82388]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T03:51:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:56 smithi195 ceph-mon[82388]: pgmap v971: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:57.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:56 smithi195 ceph-mon[82388]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T03:51:57.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:56 smithi110 ceph-mon[88262]: pgmap v971: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:57.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:56 smithi110 ceph-mon[88262]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T03:51:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:56 smithi110 ceph-mon[86423]: pgmap v971: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:56 smithi110 ceph-mon[86423]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T03:51:58.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[86423]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4222223958' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[86423]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[88262]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4222223958' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:58.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:58.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:57 smithi110 ceph-mon[88262]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T03:51:58.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:57 smithi195 ceph-mon[82388]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T03:51:58.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:57 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4222223958' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:58.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:57 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:51:58.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:58.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:57 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:51:58.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:57 smithi195 ceph-mon[82388]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T03:51:58.966 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3014 ms) 2024-09-14T03:51:58.966 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.Commit 2024-09-14T03:51:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:58 smithi110 ceph-mon[88262]: pgmap v974: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:58 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:51:58 smithi110 ceph-mon[88262]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T03:51:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:58 smithi110 ceph-mon[86423]: pgmap v974: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:58 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:59.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:51:58 smithi110 ceph-mon[86423]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T03:51:59.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:58 smithi195 ceph-mon[82388]: pgmap v974: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:59.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:58 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:59.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:51:58 smithi195 ceph-mon[82388]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T03:52:00.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:00.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:00.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[88262]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T03:52:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3878215494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:00.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:00.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:00.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:00.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[86423]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T03:52:00.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3878215494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:00.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:00 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:00.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:52:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:52:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:00 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:00 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:52:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:00 smithi195 ceph-mon[82388]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T03:52:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:00 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3878215494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:00.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:00 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:01.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:01 smithi110 ceph-mon[88262]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:01.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:01 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:01.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:01 smithi110 ceph-mon[88262]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T03:52:01.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:01 smithi110 ceph-mon[86423]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:01.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:01 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:01.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:01 smithi110 ceph-mon[86423]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T03:52:01.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:01 smithi195 ceph-mon[82388]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:01 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:01.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:01 smithi195 ceph-mon[82388]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T03:52:01.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:52:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:52:01] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.51.0" 2024-09-14T03:52:01.985 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.Commit (3019 ms) 2024-09-14T03:52:01.985 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.Abort 2024-09-14T03:52:02.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:02 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:02.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:02 smithi110 ceph-mon[88262]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T03:52:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:02 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:02.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:02 smithi110 ceph-mon[86423]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T03:52:02.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:02 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:02.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:02 smithi195 ceph-mon[82388]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T03:52:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[88262]: pgmap v980: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 724 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 399 KiB/s rd, 1.2 MiB/s wr, 1.35k op/s 2024-09-14T03:52:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:52:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[88262]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T03:52:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1474988754' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:03.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[86423]: pgmap v980: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 724 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 399 KiB/s rd, 1.2 MiB/s wr, 1.35k op/s 2024-09-14T03:52:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:52:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[86423]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T03:52:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1474988754' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:03 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:03 smithi195 ceph-mon[82388]: pgmap v980: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 724 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 399 KiB/s rd, 1.2 MiB/s wr, 1.35k op/s 2024-09-14T03:52:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:03 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:03 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:03 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:52:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:03 smithi195 ceph-mon[82388]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T03:52:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:03 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1474988754' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:03 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:04.994 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.Abort (3005 ms) 2024-09-14T03:52:04.994 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2024-09-14T03:52:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:04 smithi110 ceph-mon[88262]: pgmap v983: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:04 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:04 smithi110 ceph-mon[88262]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T03:52:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:04 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:04 smithi110 ceph-mon[86423]: pgmap v983: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:05.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:04 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:05.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:04 smithi110 ceph-mon[86423]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T03:52:05.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:04 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:04 smithi195 ceph-mon[82388]: pgmap v983: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:04 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:04 smithi195 ceph-mon[82388]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T03:52:05.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:04 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:06.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:05 smithi110 ceph-mon[88262]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T03:52:06.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:05 smithi110 ceph-mon[86423]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T03:52:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:05 smithi195 ceph-mon[82388]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T03:52:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[88262]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[88262]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T03:52:07.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2576904434' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[86423]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[86423]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T03:52:07.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:07 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2576904434' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:07.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:07 smithi195 ceph-mon[82388]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:07 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:07 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:52:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:07 smithi195 ceph-mon[82388]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T03:52:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:07 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2576904434' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:08.018 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.ReserveError (3028 ms) 2024-09-14T03:52:08.018 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.CommitError 2024-09-14T03:52:08.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:08 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2576904434' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:08.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:08 smithi110 ceph-mon[88262]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T03:52:08.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:08 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2576904434' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:08.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:08 smithi110 ceph-mon[86423]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T03:52:08.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:08 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2576904434' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:08.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:08 smithi195 ceph-mon[82388]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T03:52:09.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:09 smithi110 ceph-mon[88262]: pgmap v989: 137 pgs: 5 creating+peering, 20 unknown, 112 active+clean; 591 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 275 KiB/s rd, 275 KiB/s wr, 157 op/s 2024-09-14T03:52:09.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:09 smithi110 ceph-mon[88262]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T03:52:09.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:09 smithi110 ceph-mon[86423]: pgmap v989: 137 pgs: 5 creating+peering, 20 unknown, 112 active+clean; 591 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 275 KiB/s rd, 275 KiB/s wr, 157 op/s 2024-09-14T03:52:09.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:09 smithi110 ceph-mon[86423]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T03:52:09.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:09 smithi195 ceph-mon[82388]: pgmap v989: 137 pgs: 5 creating+peering, 20 unknown, 112 active+clean; 591 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 275 KiB/s rd, 275 KiB/s wr, 157 op/s 2024-09-14T03:52:09.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:09 smithi195 ceph-mon[82388]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T03:52:10.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:10 smithi110 ceph-mon[88262]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T03:52:10.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:10 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4132394913' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:10.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:10 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:10.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:10 smithi110 ceph-mon[86423]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T03:52:10.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:10 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4132394913' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:10.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:10 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:10.458 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:52:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:52:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:10 smithi195 ceph-mon[82388]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T03:52:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:10 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4132394913' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:10 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:11.034 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.CommitError (3015 ms) 2024-09-14T03:52:11.034 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.AbortError 2024-09-14T03:52:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-mon[88262]: pgmap v992: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-mon[88262]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T03:52:11.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-mon[86423]: pgmap v992: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-mon[86423]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T03:52:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:11 smithi195 ceph-mon[82388]: pgmap v992: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:11 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:11 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:11.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:11 smithi195 ceph-mon[82388]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T03:52:11.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:52:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:52:11] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.51.0" 2024-09-14T03:52:12.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:12 smithi110 ceph-mon[88262]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T03:52:12.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:12 smithi110 ceph-mon[86423]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T03:52:12.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:12 smithi195 ceph-mon[82388]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T03:52:13.359 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:13 smithi195 ceph-mon[82388]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:13.359 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:13 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:13.359 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:13 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:52:13.359 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:13.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:13 smithi195 ceph-mon[82388]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T03:52:13.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:13 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3361310931' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:13.360 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:13 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[88262]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[88262]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T03:52:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3361310931' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[86423]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[86423]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T03:52:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3361310931' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:13 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:14.056 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.AbortError (3021 ms) 2024-09-14T03:52:14.056 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2024-09-14T03:52:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:14 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:14.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:14 smithi110 ceph-mon[88262]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T03:52:14.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:14 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:14.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:14 smithi110 ceph-mon[86423]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T03:52:14.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:14 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:14.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:14 smithi195 ceph-mon[82388]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T03:52:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:15 smithi110 ceph-mon[88262]: pgmap v998: 137 pgs: 9 creating+peering, 19 unknown, 109 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:15 smithi110 ceph-mon[88262]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T03:52:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:15 smithi110 ceph-mon[86423]: pgmap v998: 137 pgs: 9 creating+peering, 19 unknown, 109 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:15 smithi110 ceph-mon[86423]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T03:52:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:15 smithi195 ceph-mon[82388]: pgmap v998: 137 pgs: 9 creating+peering, 19 unknown, 109 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:15 smithi195 ceph-mon[82388]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T03:52:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:16 smithi110 ceph-mon[88262]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T03:52:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:16 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3375069270' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:16 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:16.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:16 smithi110 ceph-mon[86423]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T03:52:16.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:16 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3375069270' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:16.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:16 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:16.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:16 smithi195 ceph-mon[82388]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T03:52:16.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:16 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3375069270' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:16.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:16 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:17.072 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3015 ms) 2024-09-14T03:52:17.072 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2024-09-14T03:52:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:17 smithi110 ceph-mon[88262]: pgmap v1001: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:17 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:17 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:17.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:17 smithi110 ceph-mon[88262]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T03:52:17.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:17 smithi110 ceph-mon[86423]: pgmap v1001: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:17 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:17 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:17.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:17 smithi110 ceph-mon[86423]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T03:52:17.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:17 smithi195 ceph-mon[82388]: pgmap v1001: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:17.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:17 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:17.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:17 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:17.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:17 smithi195 ceph-mon[82388]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T03:52:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:18 smithi110 ceph-mon[88262]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T03:52:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:18 smithi110 ceph-mon[86423]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T03:52:18.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:18 smithi195 ceph-mon[82388]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T03:52:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[88262]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[88262]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T03:52:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/203395200' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[86423]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[86423]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T03:52:19.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:19 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/203395200' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:19.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:19 smithi195 ceph-mon[82388]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:19.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:19 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:19.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:19 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:52:19.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:19 smithi195 ceph-mon[82388]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T03:52:19.461 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:19 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/203395200' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:20.089 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3017 ms) 2024-09-14T03:52:20.089 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2024-09-14T03:52:20.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:20 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/203395200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:20.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:20 smithi110 ceph-mon[88262]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T03:52:20.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:20 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/203395200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:20.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:20 smithi110 ceph-mon[86423]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T03:52:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:20 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/203395200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:20 smithi195 ceph-mon[82388]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T03:52:20.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:52:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:52:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:21.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:21 smithi110 ceph-mon[88262]: pgmap v1007: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:21.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:21 smithi110 ceph-mon[88262]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T03:52:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:21 smithi110 ceph-mon[86423]: pgmap v1007: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:21 smithi110 ceph-mon[86423]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T03:52:21.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:21 smithi195 ceph-mon[82388]: pgmap v1007: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:21.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:21 smithi195 ceph-mon[82388]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T03:52:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:52:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:52:21] "GET /metrics HTTP/1.1" 200 35686 "" "Prometheus/2.51.0" 2024-09-14T03:52:22.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:22 smithi110 ceph-mon[86423]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T03:52:22.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:22 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2912565665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:22.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:22 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:22.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:22 smithi110 ceph-mon[88262]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T03:52:22.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:22 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2912565665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:22.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:22 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:22.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:22 smithi195 ceph-mon[82388]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T03:52:22.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:22 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2912565665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:22.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:22 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:23.103 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3014 ms) 2024-09-14T03:52:23.103 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2024-09-14T03:52:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:23 smithi110 ceph-mon[88262]: pgmap v1010: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:23 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:23 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:23 smithi110 ceph-mon[88262]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T03:52:23.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:23 smithi110 ceph-mon[86423]: pgmap v1010: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:23.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:23 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:23.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:23 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:23.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:23 smithi110 ceph-mon[86423]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T03:52:23.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:23 smithi195 ceph-mon[82388]: pgmap v1010: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:23.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:23 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:23.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:23 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:23.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:23 smithi195 ceph-mon[82388]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T03:52:24.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:24 smithi110 ceph-mon[86423]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T03:52:24.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:24 smithi110 ceph-mon[88262]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T03:52:24.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:24 smithi195 ceph-mon[82388]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T03:52:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[88262]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[88262]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T03:52:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2894037769' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:25.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[86423]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[86423]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T03:52:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2894037769' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:25.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:25 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:25 smithi195 ceph-mon[82388]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:25 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:25 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:52:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:25 smithi195 ceph-mon[82388]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T03:52:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:25 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2894037769' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:25 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:26.121 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3019 ms) 2024-09-14T03:52:26.121 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2024-09-14T03:52:26.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:26 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:26.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:26 smithi110 ceph-mon[88262]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T03:52:26.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:26 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:26.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:26 smithi110 ceph-mon[86423]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T03:52:26.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:26 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:26.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:26 smithi195 ceph-mon[82388]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T03:52:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:27 smithi110 ceph-mon[88262]: pgmap v1016: 137 pgs: 7 creating+peering, 19 unknown, 111 active+clean; 672 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 209 KiB/s rd, 748 KiB/s wr, 743 op/s 2024-09-14T03:52:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:27 smithi110 ceph-mon[88262]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T03:52:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:27 smithi110 ceph-mon[86423]: pgmap v1016: 137 pgs: 7 creating+peering, 19 unknown, 111 active+clean; 672 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 209 KiB/s rd, 748 KiB/s wr, 743 op/s 2024-09-14T03:52:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:27 smithi110 ceph-mon[86423]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T03:52:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:27.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:27 smithi195 ceph-mon[82388]: pgmap v1016: 137 pgs: 7 creating+peering, 19 unknown, 111 active+clean; 672 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 209 KiB/s rd, 748 KiB/s wr, 743 op/s 2024-09-14T03:52:27.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:27 smithi195 ceph-mon[82388]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T03:52:27.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[88262]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T03:52:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3372166926' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[88262]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T03:52:28.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[86423]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T03:52:28.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3372166926' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:28.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:28.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:28.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:28 smithi110 ceph-mon[86423]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T03:52:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:28 smithi195 ceph-mon[82388]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T03:52:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:28 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3372166926' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:28 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:28 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:28.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:28 smithi195 ceph-mon[82388]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T03:52:29.126 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3004 ms) 2024-09-14T03:52:29.126 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2024-09-14T03:52:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:29 smithi110 ceph-mon[88262]: pgmap v1019: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:29 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:29 smithi110 ceph-mon[88262]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T03:52:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:29 smithi110 ceph-mon[86423]: pgmap v1019: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:29.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:29 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:29.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:29 smithi110 ceph-mon[86423]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T03:52:29.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:29 smithi195 ceph-mon[82388]: pgmap v1019: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:29 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:29 smithi195 ceph-mon[82388]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T03:52:30.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:30.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:30.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[88262]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T03:52:30.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4221314067' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:30.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:30.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:30.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:30.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[86423]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T03:52:30.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4221314067' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:30.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:30 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:30.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:30 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:30 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:52:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:30 smithi195 ceph-mon[82388]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T03:52:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:30 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4221314067' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:30 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:30.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:52:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:52:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:31 smithi110 ceph-mon[88262]: pgmap v1022: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:31 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:31 smithi110 ceph-mon[88262]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T03:52:31.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:31 smithi110 ceph-mon[86423]: pgmap v1022: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:31.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:31 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:31.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:31 smithi110 ceph-mon[86423]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T03:52:31.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:31 smithi195 ceph-mon[82388]: pgmap v1022: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:31.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:31 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:31.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:31 smithi195 ceph-mon[82388]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T03:52:31.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:52:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:52:31] "GET /metrics HTTP/1.1" 200 33955 "" "Prometheus/2.51.0" 2024-09-14T03:52:32.157 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3031 ms) 2024-09-14T03:52:32.157 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2024-09-14T03:52:33.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:33 smithi110 ceph-mon[88262]: pgmap v1025: 137 pgs: 2 creating+peering, 27 unknown, 108 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:33.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:33 smithi110 ceph-mon[88262]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T03:52:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:33 smithi110 ceph-mon[86423]: pgmap v1025: 137 pgs: 2 creating+peering, 27 unknown, 108 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:33 smithi110 ceph-mon[86423]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T03:52:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:33 smithi195 ceph-mon[82388]: pgmap v1025: 137 pgs: 2 creating+peering, 27 unknown, 108 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:33 smithi195 ceph-mon[82388]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T03:52:34.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:34 smithi110 ceph-mon[88262]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T03:52:34.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:34 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1275537381' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:34.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:34 smithi110 ceph-mon[86423]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T03:52:34.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:34 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1275537381' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:34.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:34 smithi195 ceph-mon[82388]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T03:52:34.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:34 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1275537381' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:35.174 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.Cleanup (3017 ms) 2024-09-14T03:52:35.174 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2024-09-14T03:52:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:35 smithi110 ceph-mon[88262]: pgmap v1028: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:35 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:35 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1275537381' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:35 smithi110 ceph-mon[88262]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T03:52:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:35 smithi110 ceph-mon[86423]: pgmap v1028: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:35 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:35 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1275537381' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:35 smithi110 ceph-mon[86423]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T03:52:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:35 smithi195 ceph-mon[82388]: pgmap v1028: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:35 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:35 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1275537381' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:35 smithi195 ceph-mon[82388]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T03:52:36.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:36 smithi110 ceph-mon[88262]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T03:52:36.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:36 smithi110 ceph-mon[86423]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T03:52:36.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:36 smithi195 ceph-mon[82388]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T03:52:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[88262]: pgmap v1031: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[88262]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T03:52:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1901020194' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[86423]: pgmap v1031: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[86423]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T03:52:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1901020194' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:37 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:37.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:37 smithi195 ceph-mon[82388]: pgmap v1031: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:37.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:37 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:37.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:37 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:52:37.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:37 smithi195 ceph-mon[82388]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T03:52:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:37 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1901020194' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:37 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:38.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:38 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:38.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:38 smithi110 ceph-mon[88262]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T03:52:38.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:38 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:38.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:38 smithi110 ceph-mon[86423]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T03:52:38.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:38 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:38.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:38 smithi195 ceph-mon[82388]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T03:52:39.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:39 smithi110 ceph-mon[88262]: pgmap v1034: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 794 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.0 MiB/s wr, 2.28k op/s 2024-09-14T03:52:39.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:39 smithi110 ceph-mon[88262]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T03:52:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:39 smithi110 ceph-mon[86423]: pgmap v1034: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 794 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.0 MiB/s wr, 2.28k op/s 2024-09-14T03:52:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:39 smithi110 ceph-mon[86423]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T03:52:39.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:39 smithi195 ceph-mon[82388]: pgmap v1034: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 794 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.0 MiB/s wr, 2.28k op/s 2024-09-14T03:52:39.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:39 smithi195 ceph-mon[82388]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T03:52:40.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:52:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:52:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:41.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:41 smithi110 ceph-mon[88262]: pgmap v1036: 137 pgs: 6 creating+peering, 4 unknown, 127 active+clean; 794 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 1.93k op/s 2024-09-14T03:52:41.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:41 smithi110 ceph-mon[86423]: pgmap v1036: 137 pgs: 6 creating+peering, 4 unknown, 127 active+clean; 794 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 1.93k op/s 2024-09-14T03:52:41.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:41 smithi195 ceph-mon[82388]: pgmap v1036: 137 pgs: 6 creating+peering, 4 unknown, 127 active+clean; 794 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 1.93k op/s 2024-09-14T03:52:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:52:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:52:41] "GET /metrics HTTP/1.1" 200 33955 "" "Prometheus/2.51.0" 2024-09-14T03:52:42.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:42.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:42.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:43.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:43 smithi110 ceph-mon[88262]: pgmap v1037: 137 pgs: 137 active+clean; 794 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 1.52k op/s 2024-09-14T03:52:43.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:43 smithi110 ceph-mon[86423]: pgmap v1037: 137 pgs: 137 active+clean; 794 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 1.52k op/s 2024-09-14T03:52:43.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:43 smithi195 ceph-mon[82388]: pgmap v1037: 137 pgs: 137 active+clean; 794 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 1.52k op/s 2024-09-14T03:52:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-14T03:52:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-14T03:52:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[86423]: pgmap v1038: 137 pgs: 137 active+clean; 2.0 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 7.6 MiB/s wr, 8.53k op/s 2024-09-14T03:52:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-14T03:52:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-14T03:52:45.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:44 smithi110 ceph-mon[88262]: pgmap v1038: 137 pgs: 137 active+clean; 2.0 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 7.6 MiB/s wr, 8.53k op/s 2024-09-14T03:52:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:44 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:44 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:44 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:44 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-14T03:52:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:44 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]} : dispatch 2024-09-14T03:52:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:44 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-14T03:52:45.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:44 smithi195 ceph-mon[82388]: pgmap v1038: 137 pgs: 137 active+clean; 2.0 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 7.6 MiB/s wr, 8.53k op/s 2024-09-14T03:52:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:45 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]}]': finished 2024-09-14T03:52:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:45 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-09-14T03:52:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:45 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-14T03:52:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:45 smithi110 ceph-mon[88262]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T03:52:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:45 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]}]': finished 2024-09-14T03:52:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:45 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-09-14T03:52:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:45 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-14T03:52:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:45 smithi110 ceph-mon[86423]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T03:52:46.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:45 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]}]': finished 2024-09-14T03:52:46.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:45 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-09-14T03:52:46.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:45 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-14T03:52:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:45 smithi195 ceph-mon[82388]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T03:52:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:46 smithi110 ceph-mon[88262]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T03:52:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:46 smithi110 ceph-mon[88262]: pgmap v1041: 137 pgs: 137 active+clean; 2.0 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 6.5 MiB/s wr, 7.36k op/s 2024-09-14T03:52:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:46 smithi110 ceph-mon[86423]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T03:52:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:46 smithi110 ceph-mon[86423]: pgmap v1041: 137 pgs: 137 active+clean; 2.0 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 6.5 MiB/s wr, 7.36k op/s 2024-09-14T03:52:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:46 smithi195 ceph-mon[82388]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T03:52:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:46 smithi195 ceph-mon[82388]: pgmap v1041: 137 pgs: 137 active+clean; 2.0 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 6.5 MiB/s wr, 7.36k op/s 2024-09-14T03:52:49.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:48 smithi110 ceph-mon[88262]: pgmap v1042: 137 pgs: 137 active+clean; 2.3 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 274 MiB/s rd, 8.3 MiB/s wr, 9.91k op/s 2024-09-14T03:52:49.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:48 smithi110 ceph-mon[86423]: pgmap v1042: 137 pgs: 137 active+clean; 2.3 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 274 MiB/s rd, 8.3 MiB/s wr, 9.91k op/s 2024-09-14T03:52:49.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:48 smithi195 ceph-mon[82388]: pgmap v1042: 137 pgs: 137 active+clean; 2.3 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 274 MiB/s rd, 8.3 MiB/s wr, 9.91k op/s 2024-09-14T03:52:49.950 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.MultiProducer (14776 ms) 2024-09-14T03:52:49.950 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2024-09-14T03:52:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:52:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:52:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:50 smithi110 ceph-mon[88262]: pgmap v1043: 137 pgs: 137 active+clean; 2.3 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 274 MiB/s rd, 8.3 MiB/s wr, 9.91k op/s 2024-09-14T03:52:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:50 smithi110 ceph-mon[88262]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T03:52:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:50 smithi110 ceph-mon[86423]: pgmap v1043: 137 pgs: 137 active+clean; 2.3 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 274 MiB/s rd, 8.3 MiB/s wr, 9.91k op/s 2024-09-14T03:52:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:50 smithi110 ceph-mon[86423]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T03:52:51.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:50 smithi195 ceph-mon[82388]: pgmap v1043: 137 pgs: 137 active+clean; 2.3 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 274 MiB/s rd, 8.3 MiB/s wr, 9.91k op/s 2024-09-14T03:52:51.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:50 smithi195 ceph-mon[82388]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T03:52:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:52:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:52:51] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.51.0" 2024-09-14T03:52:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:51 smithi195 ceph-mon[82388]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T03:52:52.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:51 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1464404757' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:52.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:51 smithi110 ceph-mon[88262]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T03:52:52.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:51 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1464404757' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:52.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:51 smithi110 ceph-mon[86423]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T03:52:52.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:51 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1464404757' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:52 smithi195 ceph-mon[82388]: pgmap v1046: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:52 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:52 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1464404757' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:52 smithi195 ceph-mon[82388]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T03:52:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:52 smithi110 ceph-mon[86423]: pgmap v1046: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:52 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:52 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1464404757' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:53.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:52 smithi110 ceph-mon[86423]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T03:52:53.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:52 smithi110 ceph-mon[88262]: pgmap v1046: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:53.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:52 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:53.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:52 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1464404757' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:53.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:52 smithi110 ceph-mon[88262]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T03:52:53.971 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (4020 ms) 2024-09-14T03:52:53.971 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2024-09-14T03:52:54.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:53 smithi110 ceph-mon[86423]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T03:52:54.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:53 smithi110 ceph-mon[88262]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T03:52:54.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:53 smithi195 ceph-mon[82388]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T03:52:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:54 smithi110 ceph-mon[86423]: pgmap v1049: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 607 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:54 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:54 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:55.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:54 smithi110 ceph-mon[86423]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T03:52:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:54 smithi110 ceph-mon[88262]: pgmap v1049: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 607 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:54 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:54 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:55.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:54 smithi110 ceph-mon[88262]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T03:52:55.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:54 smithi195 ceph-mon[82388]: pgmap v1049: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 607 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:55.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:54 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:55.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:54 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:52:55.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:54 smithi195 ceph-mon[82388]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T03:52:56.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:55 smithi110 ceph-mon[88262]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T03:52:56.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:55 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/432149844' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:56.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:55 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:56.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:55 smithi110 ceph-mon[86423]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T03:52:56.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:55 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/432149844' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:56.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:55 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:56.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:55 smithi195 ceph-mon[82388]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T03:52:56.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:55 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/432149844' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:56.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:55 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:56 smithi110 ceph-mon[86423]: pgmap v1052: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:56 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:56 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:57.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:56 smithi110 ceph-mon[86423]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T03:52:57.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:56 smithi110 ceph-mon[88262]: pgmap v1052: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:57.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:56 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:57.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:56 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:57.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:56 smithi110 ceph-mon[88262]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T03:52:57.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:56 smithi195 ceph-mon[82388]: pgmap v1052: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:57.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:56 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:57.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:56 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:57.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:56 smithi195 ceph-mon[82388]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T03:52:58.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:57 smithi110 ceph-mon[86423]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T03:52:58.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:52:58.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:58.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:57 smithi110 ceph-mon[88262]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T03:52:58.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:52:58.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:58.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:57 smithi195 ceph-mon[82388]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T03:52:58.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:52:58.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:58 smithi110 ceph-mon[88262]: pgmap v1055: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 2.9 MiB/s wr, 3.39k op/s 2024-09-14T03:52:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:58 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:59.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:52:58 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:52:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:58 smithi110 ceph-mon[86423]: pgmap v1055: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 2.9 MiB/s wr, 3.39k op/s 2024-09-14T03:52:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:58 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:59.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:52:58 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:52:59.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:58 smithi195 ceph-mon[82388]: pgmap v1055: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 2.9 MiB/s wr, 3.39k op/s 2024-09-14T03:52:59.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:58 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:59.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:52:58 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:53:00.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:53:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:53:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:01.247 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:00 smithi110 ceph-mon[88262]: pgmap v1056: 137 pgs: 4 creating+peering, 133 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 3.57k op/s 2024-09-14T03:53:01.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:00 smithi110 ceph-mon[86423]: pgmap v1056: 137 pgs: 4 creating+peering, 133 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 3.57k op/s 2024-09-14T03:53:01.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:00 smithi195 ceph-mon[82388]: pgmap v1056: 137 pgs: 4 creating+peering, 133 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 3.57k op/s 2024-09-14T03:53:01.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:53:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:53:01] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.51.0" 2024-09-14T03:53:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[88262]: pgmap v1057: 137 pgs: 137 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 3.06k op/s 2024-09-14T03:53:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:03.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[86423]: pgmap v1057: 137 pgs: 137 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 3.06k op/s 2024-09-14T03:53:03.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:03.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:02 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:03.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:02 smithi195 ceph-mon[82388]: pgmap v1057: 137 pgs: 137 active+clean; 1018 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 3.06k op/s 2024-09-14T03:53:03.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:02 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:02 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:02 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:03.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:02 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:05.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:04 smithi110 ceph-mon[88262]: pgmap v1058: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s rd, 6.4 MiB/s wr, 7.58k op/s 2024-09-14T03:53:05.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:04 smithi110 ceph-mon[86423]: pgmap v1058: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s rd, 6.4 MiB/s wr, 7.58k op/s 2024-09-14T03:53:05.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:04 smithi195 ceph-mon[82388]: pgmap v1058: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s rd, 6.4 MiB/s wr, 7.58k op/s 2024-09-14T03:53:07.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:06 smithi110 ceph-mon[88262]: pgmap v1059: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 59 MiB/s rd, 5.2 MiB/s wr, 6.12k op/s 2024-09-14T03:53:07.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:06 smithi110 ceph-mon[86423]: pgmap v1059: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 59 MiB/s rd, 5.2 MiB/s wr, 6.12k op/s 2024-09-14T03:53:07.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:06 smithi195 ceph-mon[82388]: pgmap v1059: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 59 MiB/s rd, 5.2 MiB/s wr, 6.12k op/s 2024-09-14T03:53:09.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:08 smithi110 ceph-mon[88262]: pgmap v1060: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 80 MiB/s rd, 5.9 MiB/s wr, 7.17k op/s 2024-09-14T03:53:09.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:08 smithi110 ceph-mon[86423]: pgmap v1060: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 80 MiB/s rd, 5.9 MiB/s wr, 7.17k op/s 2024-09-14T03:53:09.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:08 smithi195 ceph-mon[82388]: pgmap v1060: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 80 MiB/s rd, 5.9 MiB/s wr, 7.17k op/s 2024-09-14T03:53:10.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:53:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:53:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:11.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:10 smithi110 ceph-mon[88262]: pgmap v1061: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 3.9 MiB/s wr, 4.75k op/s 2024-09-14T03:53:11.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:10 smithi110 ceph-mon[86423]: pgmap v1061: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 3.9 MiB/s wr, 4.75k op/s 2024-09-14T03:53:11.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:10 smithi195 ceph-mon[82388]: pgmap v1061: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 3.9 MiB/s wr, 4.75k op/s 2024-09-14T03:53:11.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:53:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:53:11] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.51.0" 2024-09-14T03:53:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:13 smithi110 ceph-mon[88262]: pgmap v1062: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 3.9 MiB/s wr, 4.75k op/s 2024-09-14T03:53:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:13 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:13.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:13 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:13.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:13 smithi110 ceph-mon[86423]: pgmap v1062: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 3.9 MiB/s wr, 4.75k op/s 2024-09-14T03:53:13.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:13 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:13.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:13 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:13.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:13 smithi195 ceph-mon[82388]: pgmap v1062: 137 pgs: 137 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 3.9 MiB/s wr, 4.75k op/s 2024-09-14T03:53:13.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:13 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:13 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:14.057 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (20087 ms) 2024-09-14T03:53:14.057 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2024-09-14T03:53:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:15 smithi110 ceph-mon[88262]: pgmap v1063: 137 pgs: 137 active+clean; 1.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 115 MiB/s rd, 4.8 MiB/s wr, 6.11k op/s 2024-09-14T03:53:15.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:15 smithi110 ceph-mon[88262]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T03:53:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:15 smithi110 ceph-mon[86423]: pgmap v1063: 137 pgs: 137 active+clean; 1.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 115 MiB/s rd, 4.8 MiB/s wr, 6.11k op/s 2024-09-14T03:53:15.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:15 smithi110 ceph-mon[86423]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T03:53:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:15 smithi195 ceph-mon[82388]: pgmap v1063: 137 pgs: 137 active+clean; 1.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 115 MiB/s rd, 4.8 MiB/s wr, 6.11k op/s 2024-09-14T03:53:15.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:15 smithi195 ceph-mon[82388]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T03:53:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:16 smithi110 ceph-mon[88262]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T03:53:16.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:16 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2185556909' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:16.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:16 smithi110 ceph-mon[86423]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T03:53:16.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:16 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2185556909' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:16.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:16 smithi195 ceph-mon[82388]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T03:53:16.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:16 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2185556909' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:17.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:17 smithi110 ceph-mon[88262]: pgmap v1066: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:17.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:17 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:17.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:17 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2185556909' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:17.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:17 smithi110 ceph-mon[88262]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T03:53:17.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:17 smithi110 ceph-mon[86423]: pgmap v1066: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:17.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:17 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:17.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:17 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2185556909' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:17.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:17 smithi110 ceph-mon[86423]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T03:53:17.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:17 smithi195 ceph-mon[82388]: pgmap v1066: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:17.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:17 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:17.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:17 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2185556909' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:17.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:17 smithi195 ceph-mon[82388]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T03:53:18.078 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (4020 ms) 2024-09-14T03:53:18.078 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2024-09-14T03:53:18.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:18 smithi110 ceph-mon[88262]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T03:53:18.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:18 smithi110 ceph-mon[86423]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T03:53:18.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:18 smithi195 ceph-mon[82388]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T03:53:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:19 smithi110 ceph-mon[88262]: pgmap v1069: 137 pgs: 4 creating+peering, 18 unknown, 115 active+clean; 604 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:19 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:19 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:53:19.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:19 smithi110 ceph-mon[88262]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T03:53:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:19 smithi110 ceph-mon[86423]: pgmap v1069: 137 pgs: 4 creating+peering, 18 unknown, 115 active+clean; 604 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:19 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:19 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:53:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:19 smithi110 ceph-mon[86423]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T03:53:19.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:19 smithi195 ceph-mon[82388]: pgmap v1069: 137 pgs: 4 creating+peering, 18 unknown, 115 active+clean; 604 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:19 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:19 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:53:19.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:19 smithi195 ceph-mon[82388]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T03:53:20.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:20 smithi110 ceph-mon[88262]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T03:53:20.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:20 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4016124918' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:20.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:20 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:20.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:20 smithi110 ceph-mon[86423]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T03:53:20.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:20 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4016124918' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:20.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:20 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:20.458 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:53:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:53:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:20 smithi195 ceph-mon[82388]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T03:53:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:20 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4016124918' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:20.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:20 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-mon[86423]: pgmap v1072: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:21.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-mon[86423]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T03:53:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-mon[88262]: pgmap v1072: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:21.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-mon[88262]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T03:53:21.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:21 smithi195 ceph-mon[82388]: pgmap v1072: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:21.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:21 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:21.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:21 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:21.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:21 smithi195 ceph-mon[82388]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T03:53:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:53:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:53:21] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.51.0" 2024-09-14T03:53:22.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:22 smithi110 ceph-mon[88262]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T03:53:22.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:22 smithi110 ceph-mon[86423]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T03:53:22.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:22 smithi195 ceph-mon[82388]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T03:53:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:23 smithi110 ceph-mon[88262]: pgmap v1075: 137 pgs: 7 creating+peering, 18 unknown, 112 active+clean; 600 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:23 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:23 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:53:23.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:23 smithi110 ceph-mon[86423]: pgmap v1075: 137 pgs: 7 creating+peering, 18 unknown, 112 active+clean; 600 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:23.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:23 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:23.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:23 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:53:23.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:23 smithi195 ceph-mon[82388]: pgmap v1075: 137 pgs: 7 creating+peering, 18 unknown, 112 active+clean; 600 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:53:23.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:23 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:23.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:23 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:53:24.109 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (6031 ms) 2024-09-14T03:53:24.109 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2024-09-14T03:53:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:25 smithi110 ceph-mon[88262]: pgmap v1076: 137 pgs: 7 creating+peering, 130 active+clean; 600 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 921 KiB/s rd, 924 KiB/s wr, 278 op/s 2024-09-14T03:53:25.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:25 smithi110 ceph-mon[88262]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T03:53:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:25 smithi110 ceph-mon[86423]: pgmap v1076: 137 pgs: 7 creating+peering, 130 active+clean; 600 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 921 KiB/s rd, 924 KiB/s wr, 278 op/s 2024-09-14T03:53:25.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:25 smithi110 ceph-mon[86423]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T03:53:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:25 smithi195 ceph-mon[82388]: pgmap v1076: 137 pgs: 7 creating+peering, 130 active+clean; 600 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 921 KiB/s rd, 924 KiB/s wr, 278 op/s 2024-09-14T03:53:25.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:25 smithi195 ceph-mon[82388]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T03:53:26.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:26 smithi110 ceph-mon[88262]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T03:53:26.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:26 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/392774571' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:26.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:26 smithi110 ceph-mon[86423]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T03:53:26.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:26 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/392774571' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:26.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:26 smithi195 ceph-mon[82388]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T03:53:26.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:26 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/392774571' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[88262]: pgmap v1079: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/392774571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[88262]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T03:53:27.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:27.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[86423]: pgmap v1079: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/392774571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[86423]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T03:53:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:27.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:27 smithi195 ceph-mon[82388]: pgmap v1079: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:27.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:27 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:27.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:27 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/392774571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-116722-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:27.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:27 smithi195 ceph-mon[82388]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T03:53:27.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:27 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:27.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:28.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:28 smithi110 ceph-mon[88262]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T03:53:28.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:28 smithi110 ceph-mon[86423]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T03:53:28.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:28 smithi195 ceph-mon[82388]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T03:53:29.130 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (5021 ms) 2024-09-14T03:53:29.130 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 22 tests from TestCls2PCQueue (101908 ms total) 2024-09-14T03:53:29.130 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:53:29.130 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:53:29.130 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 22 tests from 1 test suite ran. (101908 ms total) 2024-09-14T03:53:29.130 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 22 tests. 2024-09-14T03:53:29.133 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:53:29.133 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:53:29.161 INFO:tasks.workunit:Running workunit cls/test_cls_cas.sh... 2024-09-14T03:53:29.161 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_cas.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cas.sh 2024-09-14T03:53:29.229 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:53:29.229 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 6 tests from 2 test suites. 2024-09-14T03:53:29.230 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:53:29.230 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 5 tests from cls_cas 2024-09-14T03:53:29.230 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_cas.get_put 2024-09-14T03:53:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:29 smithi110 ceph-mon[88262]: pgmap v1082: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:29 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:29.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:29 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:53:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:29 smithi110 ceph-mon[86423]: pgmap v1082: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:29.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:29 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:29.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:29 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:53:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:29 smithi195 ceph-mon[82388]: pgmap v1082: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:29 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:29.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:29 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:53:30.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:30 smithi110 ceph-mon[86423]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T03:53:30.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:30 smithi110 ceph-mon[88262]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T03:53:30.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:30 smithi195 ceph-mon[82388]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T03:53:30.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:53:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:53:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:31 smithi110 ceph-mon[86423]: pgmap v1084: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:31 smithi110 ceph-mon[86423]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T03:53:31.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:31 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1896703160' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:31.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:31 smithi110 ceph-mon[88262]: pgmap v1084: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:31.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:31 smithi110 ceph-mon[88262]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T03:53:31.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:31 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1896703160' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:31.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:31 smithi195 ceph-mon[82388]: pgmap v1084: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:31.476 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:31 smithi195 ceph-mon[82388]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T03:53:31.476 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:31 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1896703160' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:31.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:53:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:53:31] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-09-14T03:53:32.157 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_cas.get_put (2925 ms) 2024-09-14T03:53:32.157 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_cas.wrong_put 2024-09-14T03:53:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:32 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1896703160' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:32.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:32 smithi110 ceph-mon[86423]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T03:53:32.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:32 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1896703160' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:32.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:32 smithi110 ceph-mon[88262]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T03:53:32.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:32 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1896703160' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:32.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:32 smithi195 ceph-mon[82388]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T03:53:33.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:33 smithi110 ceph-mon[88262]: pgmap v1087: 137 pgs: 5 creating+peering, 24 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:33.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:33 smithi110 ceph-mon[88262]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T03:53:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:33 smithi110 ceph-mon[86423]: pgmap v1087: 137 pgs: 5 creating+peering, 24 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:33.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:33 smithi110 ceph-mon[86423]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T03:53:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:33 smithi195 ceph-mon[82388]: pgmap v1087: 137 pgs: 5 creating+peering, 24 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:33.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:33 smithi195 ceph-mon[82388]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T03:53:34.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:34 smithi110 ceph-mon[88262]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T03:53:34.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:34 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/389981726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:34.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:34 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:34.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:34 smithi110 ceph-mon[86423]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T03:53:34.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:34 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/389981726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:34.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:34 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:34 smithi195 ceph-mon[82388]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T03:53:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:34 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/389981726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:34.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:34 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:35.171 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_cas.wrong_put (3017 ms) 2024-09-14T03:53:35.171 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_cas.dup_get 2024-09-14T03:53:35.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:35 smithi110 ceph-mon[88262]: pgmap v1090: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:35.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:35 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:35.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:35 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:35.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:35 smithi110 ceph-mon[88262]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T03:53:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:35 smithi110 ceph-mon[86423]: pgmap v1090: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:35 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:35 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:35.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:35 smithi110 ceph-mon[86423]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T03:53:35.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:35 smithi195 ceph-mon[82388]: pgmap v1090: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:35 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:35 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:35.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:35 smithi195 ceph-mon[82388]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T03:53:36.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:36 smithi110 ceph-mon[88262]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T03:53:36.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:36 smithi110 ceph-mon[86423]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T03:53:36.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:36 smithi195 ceph-mon[82388]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T03:53:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[88262]: pgmap v1093: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:37.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:37.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:53:37.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[88262]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T03:53:37.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2611686238' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:37.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[86423]: pgmap v1093: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:37.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:37.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:53:37.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[86423]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T03:53:37.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2611686238' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:37.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:37 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:37 smithi195 ceph-mon[82388]: pgmap v1093: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:37 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:37 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:53:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:37 smithi195 ceph-mon[82388]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T03:53:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:37 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2611686238' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:37.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:37 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:38.192 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_cas.dup_get (3022 ms) 2024-09-14T03:53:38.193 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_cas.dup_put 2024-09-14T03:53:38.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:38 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:38.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:38 smithi110 ceph-mon[88262]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T03:53:38.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:38 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:38.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:38 smithi110 ceph-mon[86423]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T03:53:38.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:38 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:38.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:38 smithi195 ceph-mon[82388]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T03:53:39.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:39 smithi110 ceph-mon[88262]: pgmap v1096: 137 pgs: 1 creating+peering, 18 unknown, 118 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:39.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:39 smithi110 ceph-mon[88262]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T03:53:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:39 smithi110 ceph-mon[86423]: pgmap v1096: 137 pgs: 1 creating+peering, 18 unknown, 118 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:39.446 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:39 smithi110 ceph-mon[86423]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T03:53:39.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:39 smithi195 ceph-mon[82388]: pgmap v1096: 137 pgs: 1 creating+peering, 18 unknown, 118 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:39.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:39 smithi195 ceph-mon[82388]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T03:53:40.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:40 smithi195 ceph-mon[82388]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T03:53:40.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:40 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2124379048' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:40.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:53:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:53:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:40.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:40 smithi110 ceph-mon[86423]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T03:53:40.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:40 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2124379048' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:40.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:40 smithi110 ceph-mon[88262]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T03:53:40.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:40 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2124379048' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:41.209 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_cas.dup_put (3017 ms) 2024-09-14T03:53:41.209 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_cas.get_wrong_data 2024-09-14T03:53:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:41 smithi195 ceph-mon[82388]: pgmap v1099: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:41 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:41 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2124379048' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:41.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:41 smithi195 ceph-mon[82388]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T03:53:41.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-mon[88262]: pgmap v1099: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:41.563 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:41.563 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2124379048' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:41.563 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-mon[88262]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T03:53:41.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-mon[86423]: pgmap v1099: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:41.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:41.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2124379048' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:41.563 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-mon[86423]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T03:53:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:53:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:53:41] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-09-14T03:53:42.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:42 smithi110 ceph-mon[86423]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T03:53:42.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:42.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:42 smithi110 ceph-mon[88262]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T03:53:42.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:42.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:42 smithi195 ceph-mon[82388]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T03:53:42.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:43.610 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:43 smithi195 ceph-mon[82388]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:43.610 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:43 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:43.610 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:43 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:53:43.610 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:43 smithi195 ceph-mon[82388]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T03:53:43.611 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:43 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4012188054' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:43.611 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:43 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:43.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[86423]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:43.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:43.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:53:43.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[86423]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T03:53:43.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4012188054' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:43.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:43.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[88262]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:43.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:43.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:53:43.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[88262]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T03:53:43.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4012188054' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:43.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:43 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:44.227 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_cas.get_wrong_data (3017 ms) 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 5 tests from cls_cas (14998 ms total) 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 1 test from chunk_refs_t 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] chunk_refs_t.size 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:1 60 by_object 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:2 103 by_object 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:4 189 by_object 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:8 361 by_object 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:16 112 by_hash(32 bits) 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:32 208 by_hash(32 bits) 2024-09-14T03:53:44.228 INFO:tasks.workunit.client.0.smithi110.stdout:64 400 by_hash(32 bits) 2024-09-14T03:53:44.229 INFO:tasks.workunit.client.0.smithi110.stdout:128 509 by_hash(10 bits) 2024-09-14T03:53:44.229 INFO:tasks.workunit.client.0.smithi110.stdout:256 499 by_hash(7 bits) 2024-09-14T03:53:44.229 INFO:tasks.workunit.client.0.smithi110.stdout:512 462 by_hash(5 bits) 2024-09-14T03:53:44.229 INFO:tasks.workunit.client.0.smithi110.stdout:1024 466 by_hash(5 bits) 2024-09-14T03:53:44.232 INFO:tasks.workunit.client.0.smithi110.stdout:2048 466 by_hash(5 bits) 2024-09-14T03:53:44.232 INFO:tasks.workunit.client.0.smithi110.stdout:4096 470 by_hash(5 bits) 2024-09-14T03:53:44.232 INFO:tasks.workunit.client.0.smithi110.stdout:8192 498 by_hash(4 bits) 2024-09-14T03:53:44.232 INFO:tasks.workunit.client.0.smithi110.stdout:16384 278 by_hash(3 bits) 2024-09-14T03:53:44.234 INFO:tasks.workunit.client.0.smithi110.stdout:32768 337 by_hash(2 bits) 2024-09-14T03:53:44.238 INFO:tasks.workunit.client.0.smithi110.stdout:65536 367 by_hash(2 bits) 2024-09-14T03:53:44.245 INFO:tasks.workunit.client.0.smithi110.stdout:131072 387 by_hash(1 bits) 2024-09-14T03:53:44.259 INFO:tasks.workunit.client.0.smithi110.stdout:262144 331 by_pool 2024-09-14T03:53:44.287 INFO:tasks.workunit.client.0.smithi110.stdout:524288 21 count 2024-09-14T03:53:44.335 INFO:tasks.workunit.client.0.smithi110.stdout:1048576 21 count 2024-09-14T03:53:44.440 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] chunk_refs_t.size (214 ms) 2024-09-14T03:53:44.441 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 1 test from chunk_refs_t (214 ms total) 2024-09-14T03:53:44.441 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:53:44.441 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:53:44.441 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 6 tests from 2 test suites ran. (15212 ms total) 2024-09-14T03:53:44.441 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 6 tests. 2024-09-14T03:53:44.443 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:53:44.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:53:44.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:44 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:44.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:44 smithi110 ceph-mon[88262]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T03:53:44.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:44 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:44.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:44 smithi110 ceph-mon[86423]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T03:53:44.508 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2024-09-14T03:53:44.508 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2024-09-14T03:53:44.575 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:53:44.575 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 31 tests from 1 test suite. 2024-09-14T03:53:44.575 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:53:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:44 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117419-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:44.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:44 smithi195 ceph-mon[82388]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T03:53:45.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:45 smithi110 ceph-mon[88262]: pgmap v1105: 137 pgs: 9 creating+peering, 14 unknown, 114 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:45.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:45 smithi110 ceph-mon[88262]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T03:53:45.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:45 smithi110 ceph-mon[86423]: pgmap v1105: 137 pgs: 9 creating+peering, 14 unknown, 114 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:45.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:45 smithi110 ceph-mon[86423]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T03:53:45.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:45 smithi195 ceph-mon[82388]: pgmap v1105: 137 pgs: 9 creating+peering, 14 unknown, 114 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:45.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:45 smithi195 ceph-mon[82388]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T03:53:46.235 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 31 tests from CmpOmap 2024-09-14T03:53:46.236 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2024-09-14T03:53:46.252 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (16 ms) 2024-09-14T03:53:46.252 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2024-09-14T03:53:46.256 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (5 ms) 2024-09-14T03:53:46.256 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2024-09-14T03:53:46.259 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (3 ms) 2024-09-14T03:53:46.259 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2024-09-14T03:53:46.262 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (3 ms) 2024-09-14T03:53:46.262 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_str 2024-09-14T03:53:46.268 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_str (5 ms) 2024-09-14T03:53:46.268 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2024-09-14T03:53:46.274 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_u64 (6 ms) 2024-09-14T03:53:46.274 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2024-09-14T03:53:46.276 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (3 ms) 2024-09-14T03:53:46.277 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2024-09-14T03:53:46.280 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (3 ms) 2024-09-14T03:53:46.280 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2024-09-14T03:53:46.281 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (2 ms) 2024-09-14T03:53:46.282 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2024-09-14T03:53:46.286 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (4 ms) 2024-09-14T03:53:46.286 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2024-09-14T03:53:46.287 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2024-09-14T03:53:46.287 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2024-09-14T03:53:46.287 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (1 ms) 2024-09-14T03:53:46.287 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2024-09-14T03:53:46.293 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (5 ms) 2024-09-14T03:53:46.293 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2024-09-14T03:53:46.298 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (6 ms) 2024-09-14T03:53:46.299 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2024-09-14T03:53:46.304 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (6 ms) 2024-09-14T03:53:46.304 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2024-09-14T03:53:46.311 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (7 ms) 2024-09-14T03:53:46.311 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2024-09-14T03:53:46.318 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_str (7 ms) 2024-09-14T03:53:46.318 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2024-09-14T03:53:46.324 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (6 ms) 2024-09-14T03:53:46.325 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2024-09-14T03:53:46.326 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (2 ms) 2024-09-14T03:53:46.326 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2024-09-14T03:53:46.328 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (2 ms) 2024-09-14T03:53:46.328 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2024-09-14T03:53:46.329 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (1 ms) 2024-09-14T03:53:46.330 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2024-09-14T03:53:46.330 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (1 ms) 2024-09-14T03:53:46.330 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2024-09-14T03:53:46.336 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (6 ms) 2024-09-14T03:53:46.336 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2024-09-14T03:53:46.342 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (5 ms) 2024-09-14T03:53:46.342 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2024-09-14T03:53:46.349 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (7 ms) 2024-09-14T03:53:46.349 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2024-09-14T03:53:46.355 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (7 ms) 2024-09-14T03:53:46.355 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2024-09-14T03:53:46.357 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (2 ms) 2024-09-14T03:53:46.357 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2024-09-14T03:53:46.359 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (2 ms) 2024-09-14T03:53:46.359 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2024-09-14T03:53:46.360 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (1 ms) 2024-09-14T03:53:46.360 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2024-09-14T03:53:46.360 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (0 ms) 2024-09-14T03:53:46.361 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2024-09-14T03:53:46.368 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (7 ms) 2024-09-14T03:53:46.368 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 31 tests from CmpOmap (132 ms total) 2024-09-14T03:53:46.368 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:53:46.368 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:53:46.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:46 smithi110 ceph-mon[88262]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T03:53:46.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:46 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1404641039' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117542-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:46.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:46 smithi110 ceph-mon[86423]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T03:53:46.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:46 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1404641039' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117542-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:46.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:46 smithi195 ceph-mon[82388]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T03:53:46.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:46 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1404641039' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117542-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:47.241 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 31 tests from 1 test suite ran. (2667 ms total) 2024-09-14T03:53:47.242 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 31 tests. 2024-09-14T03:53:47.244 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:53:47.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:53:47.312 INFO:tasks.workunit:Running workunit cls/test_cls_hello.sh... 2024-09-14T03:53:47.313 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_hello.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_hello.sh 2024-09-14T03:53:47.381 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:53:47.381 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 6 tests from 1 test suite. 2024-09-14T03:53:47.381 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:53:47.381 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 6 tests from ClsHello 2024-09-14T03:53:47.381 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsHello.SayHello 2024-09-14T03:53:47.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:47 smithi110 ceph-mon[88262]: pgmap v1108: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:47.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:47 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:47.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:47 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1404641039' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117542-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:47.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:47 smithi110 ceph-mon[88262]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T03:53:47.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:47 smithi110 ceph-mon[86423]: pgmap v1108: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:47.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:47 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:47.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:47 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1404641039' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117542-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:47.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:47 smithi110 ceph-mon[86423]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T03:53:47.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:47 smithi195 ceph-mon[82388]: pgmap v1108: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:47.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:47 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:47.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:47 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1404641039' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117542-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:47.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:47 smithi195 ceph-mon[82388]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T03:53:48.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:48 smithi110 ceph-mon[88262]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T03:53:48.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:48 smithi110 ceph-mon[86423]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T03:53:48.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:48 smithi195 ceph-mon[82388]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T03:53:49.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[88262]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:49.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:49.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:53:49.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[88262]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T03:53:49.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1206560095' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:49.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[86423]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:53:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[86423]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T03:53:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1206560095' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:49.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:49 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:49 smithi195 ceph-mon[82388]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:49 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:49 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:53:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:49 smithi195 ceph-mon[82388]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T03:53:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:49 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1206560095' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:49.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:49 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:50.261 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsHello.SayHello (2879 ms) 2024-09-14T03:53:50.261 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsHello.RecordHello 2024-09-14T03:53:50.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:50 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:50.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:50 smithi110 ceph-mon[86423]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T03:53:50.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:50 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:50.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:50 smithi110 ceph-mon[88262]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T03:53:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:50 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:50.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:50 smithi195 ceph-mon[82388]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T03:53:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:53:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:53:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:51.562 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:51 smithi110 ceph-mon[86423]: pgmap v1114: 137 pgs: 12 creating+peering, 17 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:51.562 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:51 smithi110 ceph-mon[86423]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T03:53:51.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:51 smithi110 ceph-mon[88262]: pgmap v1114: 137 pgs: 12 creating+peering, 17 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:51.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:51 smithi110 ceph-mon[88262]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T03:53:51.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:51 smithi195 ceph-mon[82388]: pgmap v1114: 137 pgs: 12 creating+peering, 17 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:51.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:51 smithi195 ceph-mon[82388]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T03:53:51.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:53:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:53:51] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.51.0" 2024-09-14T03:53:52.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:52 smithi110 ceph-mon[86423]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T03:53:52.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:52 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/737420713' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:52.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:52 smithi110 ceph-mon[88262]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T03:53:52.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:52 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/737420713' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:52.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:52 smithi195 ceph-mon[82388]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T03:53:52.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:52 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/737420713' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:53.273 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsHello.RecordHello (3012 ms) 2024-09-14T03:53:53.273 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsHello.WriteReturnData 2024-09-14T03:53:53.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:53 smithi110 ceph-mon[86423]: pgmap v1117: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:53.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:53 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:53.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:53 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/737420713' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:53.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:53 smithi110 ceph-mon[86423]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T03:53:53.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:53 smithi110 ceph-mon[88262]: pgmap v1117: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:53.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:53 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:53.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:53 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/737420713' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:53.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:53 smithi110 ceph-mon[88262]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T03:53:53.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:53 smithi195 ceph-mon[82388]: pgmap v1117: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:53.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:53 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:53 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/737420713' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:53.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:53 smithi195 ceph-mon[82388]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T03:53:54.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:54 smithi110 ceph-mon[86423]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T03:53:54.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:54 smithi110 ceph-mon[88262]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T03:53:54.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:54 smithi195 ceph-mon[82388]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T03:53:55.292 INFO:tasks.workunit.client.0.smithi110.stdout:require_osd_release = squid 2024-09-14T03:53:55.298 INFO:tasks.workunit.client.0.smithi110.stdout:0 2024-09-14T03:53:55.475 INFO:tasks.workunit.client.0.smithi110.stdout:1 2024-09-14T03:53:55.476 INFO:tasks.workunit.client.0.smithi110.stdout:2 2024-09-14T03:53:55.477 INFO:tasks.workunit.client.0.smithi110.stdout:3 2024-09-14T03:53:55.478 INFO:tasks.workunit.client.0.smithi110.stdout:4 2024-09-14T03:53:55.479 INFO:tasks.workunit.client.0.smithi110.stdout:5 2024-09-14T03:53:55.480 INFO:tasks.workunit.client.0.smithi110.stdout:6 2024-09-14T03:53:55.484 INFO:tasks.workunit.client.0.smithi110.stdout:7 2024-09-14T03:53:55.485 INFO:tasks.workunit.client.0.smithi110.stdout:8 2024-09-14T03:53:55.485 INFO:tasks.workunit.client.0.smithi110.stdout:9 2024-09-14T03:53:55.485 INFO:tasks.workunit.client.0.smithi110.stdout:10 2024-09-14T03:53:55.485 INFO:tasks.workunit.client.0.smithi110.stdout:11 2024-09-14T03:53:55.485 INFO:tasks.workunit.client.0.smithi110.stdout:12 2024-09-14T03:53:55.487 INFO:tasks.workunit.client.0.smithi110.stdout:13 2024-09-14T03:53:55.487 INFO:tasks.workunit.client.0.smithi110.stdout:14 2024-09-14T03:53:55.488 INFO:tasks.workunit.client.0.smithi110.stdout:15 2024-09-14T03:53:55.489 INFO:tasks.workunit.client.0.smithi110.stdout:16 2024-09-14T03:53:55.490 INFO:tasks.workunit.client.0.smithi110.stdout:17 2024-09-14T03:53:55.491 INFO:tasks.workunit.client.0.smithi110.stdout:18 2024-09-14T03:53:55.492 INFO:tasks.workunit.client.0.smithi110.stdout:19 2024-09-14T03:53:55.493 INFO:tasks.workunit.client.0.smithi110.stdout:20 2024-09-14T03:53:55.494 INFO:tasks.workunit.client.0.smithi110.stdout:21 2024-09-14T03:53:55.495 INFO:tasks.workunit.client.0.smithi110.stdout:22 2024-09-14T03:53:55.496 INFO:tasks.workunit.client.0.smithi110.stdout:23 2024-09-14T03:53:55.497 INFO:tasks.workunit.client.0.smithi110.stdout:24 2024-09-14T03:53:55.498 INFO:tasks.workunit.client.0.smithi110.stdout:25 2024-09-14T03:53:55.499 INFO:tasks.workunit.client.0.smithi110.stdout:26 2024-09-14T03:53:55.500 INFO:tasks.workunit.client.0.smithi110.stdout:27 2024-09-14T03:53:55.501 INFO:tasks.workunit.client.0.smithi110.stdout:28 2024-09-14T03:53:55.502 INFO:tasks.workunit.client.0.smithi110.stdout:29 2024-09-14T03:53:55.503 INFO:tasks.workunit.client.0.smithi110.stdout:30 2024-09-14T03:53:55.503 INFO:tasks.workunit.client.0.smithi110.stdout:31 2024-09-14T03:53:55.504 INFO:tasks.workunit.client.0.smithi110.stdout:32 2024-09-14T03:53:55.505 INFO:tasks.workunit.client.0.smithi110.stdout:33 2024-09-14T03:53:55.506 INFO:tasks.workunit.client.0.smithi110.stdout:34 2024-09-14T03:53:55.507 INFO:tasks.workunit.client.0.smithi110.stdout:35 2024-09-14T03:53:55.508 INFO:tasks.workunit.client.0.smithi110.stdout:36 2024-09-14T03:53:55.509 INFO:tasks.workunit.client.0.smithi110.stdout:37 2024-09-14T03:53:55.509 INFO:tasks.workunit.client.0.smithi110.stdout:38 2024-09-14T03:53:55.510 INFO:tasks.workunit.client.0.smithi110.stdout:39 2024-09-14T03:53:55.511 INFO:tasks.workunit.client.0.smithi110.stdout:40 2024-09-14T03:53:55.512 INFO:tasks.workunit.client.0.smithi110.stdout:41 2024-09-14T03:53:55.513 INFO:tasks.workunit.client.0.smithi110.stdout:42 2024-09-14T03:53:55.514 INFO:tasks.workunit.client.0.smithi110.stdout:43 2024-09-14T03:53:55.514 INFO:tasks.workunit.client.0.smithi110.stdout:44 2024-09-14T03:53:55.515 INFO:tasks.workunit.client.0.smithi110.stdout:45 2024-09-14T03:53:55.516 INFO:tasks.workunit.client.0.smithi110.stdout:46 2024-09-14T03:53:55.517 INFO:tasks.workunit.client.0.smithi110.stdout:47 2024-09-14T03:53:55.518 INFO:tasks.workunit.client.0.smithi110.stdout:48 2024-09-14T03:53:55.519 INFO:tasks.workunit.client.0.smithi110.stdout:49 2024-09-14T03:53:55.520 INFO:tasks.workunit.client.0.smithi110.stdout:50 2024-09-14T03:53:55.521 INFO:tasks.workunit.client.0.smithi110.stdout:51 2024-09-14T03:53:55.522 INFO:tasks.workunit.client.0.smithi110.stdout:52 2024-09-14T03:53:55.522 INFO:tasks.workunit.client.0.smithi110.stdout:53 2024-09-14T03:53:55.523 INFO:tasks.workunit.client.0.smithi110.stdout:54 2024-09-14T03:53:55.524 INFO:tasks.workunit.client.0.smithi110.stdout:55 2024-09-14T03:53:55.525 INFO:tasks.workunit.client.0.smithi110.stdout:56 2024-09-14T03:53:55.526 INFO:tasks.workunit.client.0.smithi110.stdout:57 2024-09-14T03:53:55.527 INFO:tasks.workunit.client.0.smithi110.stdout:58 2024-09-14T03:53:55.528 INFO:tasks.workunit.client.0.smithi110.stdout:59 2024-09-14T03:53:55.529 INFO:tasks.workunit.client.0.smithi110.stdout:60 2024-09-14T03:53:55.530 INFO:tasks.workunit.client.0.smithi110.stdout:61 2024-09-14T03:53:55.530 INFO:tasks.workunit.client.0.smithi110.stdout:62 2024-09-14T03:53:55.531 INFO:tasks.workunit.client.0.smithi110.stdout:63 2024-09-14T03:53:55.532 INFO:tasks.workunit.client.0.smithi110.stdout:64 2024-09-14T03:53:55.533 INFO:tasks.workunit.client.0.smithi110.stdout:65 2024-09-14T03:53:55.534 INFO:tasks.workunit.client.0.smithi110.stdout:66 2024-09-14T03:53:55.535 INFO:tasks.workunit.client.0.smithi110.stdout:67 2024-09-14T03:53:55.536 INFO:tasks.workunit.client.0.smithi110.stdout:68 2024-09-14T03:53:55.536 INFO:tasks.workunit.client.0.smithi110.stdout:69 2024-09-14T03:53:55.537 INFO:tasks.workunit.client.0.smithi110.stdout:70 2024-09-14T03:53:55.538 INFO:tasks.workunit.client.0.smithi110.stdout:71 2024-09-14T03:53:55.539 INFO:tasks.workunit.client.0.smithi110.stdout:72 2024-09-14T03:53:55.540 INFO:tasks.workunit.client.0.smithi110.stdout:73 2024-09-14T03:53:55.541 INFO:tasks.workunit.client.0.smithi110.stdout:74 2024-09-14T03:53:55.542 INFO:tasks.workunit.client.0.smithi110.stdout:75 2024-09-14T03:53:55.542 INFO:tasks.workunit.client.0.smithi110.stdout:76 2024-09-14T03:53:55.543 INFO:tasks.workunit.client.0.smithi110.stdout:77 2024-09-14T03:53:55.544 INFO:tasks.workunit.client.0.smithi110.stdout:78 2024-09-14T03:53:55.545 INFO:tasks.workunit.client.0.smithi110.stdout:79 2024-09-14T03:53:55.546 INFO:tasks.workunit.client.0.smithi110.stdout:80 2024-09-14T03:53:55.546 INFO:tasks.workunit.client.0.smithi110.stdout:81 2024-09-14T03:53:55.547 INFO:tasks.workunit.client.0.smithi110.stdout:82 2024-09-14T03:53:55.548 INFO:tasks.workunit.client.0.smithi110.stdout:83 2024-09-14T03:53:55.549 INFO:tasks.workunit.client.0.smithi110.stdout:84 2024-09-14T03:53:55.550 INFO:tasks.workunit.client.0.smithi110.stdout:85 2024-09-14T03:53:55.550 INFO:tasks.workunit.client.0.smithi110.stdout:86 2024-09-14T03:53:55.551 INFO:tasks.workunit.client.0.smithi110.stdout:87 2024-09-14T03:53:55.552 INFO:tasks.workunit.client.0.smithi110.stdout:88 2024-09-14T03:53:55.553 INFO:tasks.workunit.client.0.smithi110.stdout:89 2024-09-14T03:53:55.554 INFO:tasks.workunit.client.0.smithi110.stdout:90 2024-09-14T03:53:55.555 INFO:tasks.workunit.client.0.smithi110.stdout:91 2024-09-14T03:53:55.555 INFO:tasks.workunit.client.0.smithi110.stdout:92 2024-09-14T03:53:55.556 INFO:tasks.workunit.client.0.smithi110.stdout:93 2024-09-14T03:53:55.557 INFO:tasks.workunit.client.0.smithi110.stdout:94 2024-09-14T03:53:55.558 INFO:tasks.workunit.client.0.smithi110.stdout:95 2024-09-14T03:53:55.559 INFO:tasks.workunit.client.0.smithi110.stdout:96 2024-09-14T03:53:55.559 INFO:tasks.workunit.client.0.smithi110.stdout:97 2024-09-14T03:53:55.560 INFO:tasks.workunit.client.0.smithi110.stdout:98 2024-09-14T03:53:55.561 INFO:tasks.workunit.client.0.smithi110.stdout:99 2024-09-14T03:53:55.562 INFO:tasks.workunit.client.0.smithi110.stdout:100 2024-09-14T03:53:55.563 INFO:tasks.workunit.client.0.smithi110.stdout:101 2024-09-14T03:53:55.564 INFO:tasks.workunit.client.0.smithi110.stdout:102 2024-09-14T03:53:55.564 INFO:tasks.workunit.client.0.smithi110.stdout:103 2024-09-14T03:53:55.565 INFO:tasks.workunit.client.0.smithi110.stdout:104 2024-09-14T03:53:55.566 INFO:tasks.workunit.client.0.smithi110.stdout:105 2024-09-14T03:53:55.567 INFO:tasks.workunit.client.0.smithi110.stdout:106 2024-09-14T03:53:55.568 INFO:tasks.workunit.client.0.smithi110.stdout:107 2024-09-14T03:53:55.568 INFO:tasks.workunit.client.0.smithi110.stdout:108 2024-09-14T03:53:55.569 INFO:tasks.workunit.client.0.smithi110.stdout:109 2024-09-14T03:53:55.570 INFO:tasks.workunit.client.0.smithi110.stdout:110 2024-09-14T03:53:55.571 INFO:tasks.workunit.client.0.smithi110.stdout:111 2024-09-14T03:53:55.572 INFO:tasks.workunit.client.0.smithi110.stdout:112 2024-09-14T03:53:55.573 INFO:tasks.workunit.client.0.smithi110.stdout:113 2024-09-14T03:53:55.573 INFO:tasks.workunit.client.0.smithi110.stdout:114 2024-09-14T03:53:55.574 INFO:tasks.workunit.client.0.smithi110.stdout:115 2024-09-14T03:53:55.575 INFO:tasks.workunit.client.0.smithi110.stdout:116 2024-09-14T03:53:55.576 INFO:tasks.workunit.client.0.smithi110.stdout:117 2024-09-14T03:53:55.577 INFO:tasks.workunit.client.0.smithi110.stdout:118 2024-09-14T03:53:55.577 INFO:tasks.workunit.client.0.smithi110.stdout:119 2024-09-14T03:53:55.578 INFO:tasks.workunit.client.0.smithi110.stdout:120 2024-09-14T03:53:55.579 INFO:tasks.workunit.client.0.smithi110.stdout:121 2024-09-14T03:53:55.580 INFO:tasks.workunit.client.0.smithi110.stdout:122 2024-09-14T03:53:55.581 INFO:tasks.workunit.client.0.smithi110.stdout:123 2024-09-14T03:53:55.581 INFO:tasks.workunit.client.0.smithi110.stdout:124 2024-09-14T03:53:55.582 INFO:tasks.workunit.client.0.smithi110.stdout:125 2024-09-14T03:53:55.583 INFO:tasks.workunit.client.0.smithi110.stdout:126 2024-09-14T03:53:55.584 INFO:tasks.workunit.client.0.smithi110.stdout:127 2024-09-14T03:53:55.585 INFO:tasks.workunit.client.0.smithi110.stdout:128 2024-09-14T03:53:55.586 INFO:tasks.workunit.client.0.smithi110.stdout:129 2024-09-14T03:53:55.586 INFO:tasks.workunit.client.0.smithi110.stdout:130 2024-09-14T03:53:55.587 INFO:tasks.workunit.client.0.smithi110.stdout:131 2024-09-14T03:53:55.588 INFO:tasks.workunit.client.0.smithi110.stdout:132 2024-09-14T03:53:55.589 INFO:tasks.workunit.client.0.smithi110.stdout:133 2024-09-14T03:53:55.590 INFO:tasks.workunit.client.0.smithi110.stdout:134 2024-09-14T03:53:55.591 INFO:tasks.workunit.client.0.smithi110.stdout:135 2024-09-14T03:53:55.591 INFO:tasks.workunit.client.0.smithi110.stdout:136 2024-09-14T03:53:55.592 INFO:tasks.workunit.client.0.smithi110.stdout:137 2024-09-14T03:53:55.593 INFO:tasks.workunit.client.0.smithi110.stdout:138 2024-09-14T03:53:55.594 INFO:tasks.workunit.client.0.smithi110.stdout:139 2024-09-14T03:53:55.594 INFO:tasks.workunit.client.0.smithi110.stdout:140 2024-09-14T03:53:55.595 INFO:tasks.workunit.client.0.smithi110.stdout:141 2024-09-14T03:53:55.596 INFO:tasks.workunit.client.0.smithi110.stdout:142 2024-09-14T03:53:55.597 INFO:tasks.workunit.client.0.smithi110.stdout:143 2024-09-14T03:53:55.598 INFO:tasks.workunit.client.0.smithi110.stdout:144 2024-09-14T03:53:55.598 INFO:tasks.workunit.client.0.smithi110.stdout:145 2024-09-14T03:53:55.599 INFO:tasks.workunit.client.0.smithi110.stdout:146 2024-09-14T03:53:55.600 INFO:tasks.workunit.client.0.smithi110.stdout:147 2024-09-14T03:53:55.601 INFO:tasks.workunit.client.0.smithi110.stdout:148 2024-09-14T03:53:55.601 INFO:tasks.workunit.client.0.smithi110.stdout:149 2024-09-14T03:53:55.602 INFO:tasks.workunit.client.0.smithi110.stdout:150 2024-09-14T03:53:55.603 INFO:tasks.workunit.client.0.smithi110.stdout:151 2024-09-14T03:53:55.604 INFO:tasks.workunit.client.0.smithi110.stdout:152 2024-09-14T03:53:55.605 INFO:tasks.workunit.client.0.smithi110.stdout:153 2024-09-14T03:53:55.606 INFO:tasks.workunit.client.0.smithi110.stdout:154 2024-09-14T03:53:55.606 INFO:tasks.workunit.client.0.smithi110.stdout:155 2024-09-14T03:53:55.607 INFO:tasks.workunit.client.0.smithi110.stdout:156 2024-09-14T03:53:55.608 INFO:tasks.workunit.client.0.smithi110.stdout:157 2024-09-14T03:53:55.609 INFO:tasks.workunit.client.0.smithi110.stdout:158 2024-09-14T03:53:55.610 INFO:tasks.workunit.client.0.smithi110.stdout:159 2024-09-14T03:53:55.610 INFO:tasks.workunit.client.0.smithi110.stdout:160 2024-09-14T03:53:55.611 INFO:tasks.workunit.client.0.smithi110.stdout:161 2024-09-14T03:53:55.612 INFO:tasks.workunit.client.0.smithi110.stdout:162 2024-09-14T03:53:55.613 INFO:tasks.workunit.client.0.smithi110.stdout:163 2024-09-14T03:53:55.614 INFO:tasks.workunit.client.0.smithi110.stdout:164 2024-09-14T03:53:55.615 INFO:tasks.workunit.client.0.smithi110.stdout:165 2024-09-14T03:53:55.615 INFO:tasks.workunit.client.0.smithi110.stdout:166 2024-09-14T03:53:55.616 INFO:tasks.workunit.client.0.smithi110.stdout:167 2024-09-14T03:53:55.617 INFO:tasks.workunit.client.0.smithi110.stdout:168 2024-09-14T03:53:55.617 INFO:tasks.workunit.client.0.smithi110.stdout:169 2024-09-14T03:53:55.618 INFO:tasks.workunit.client.0.smithi110.stdout:170 2024-09-14T03:53:55.619 INFO:tasks.workunit.client.0.smithi110.stdout:171 2024-09-14T03:53:55.620 INFO:tasks.workunit.client.0.smithi110.stdout:172 2024-09-14T03:53:55.621 INFO:tasks.workunit.client.0.smithi110.stdout:173 2024-09-14T03:53:55.622 INFO:tasks.workunit.client.0.smithi110.stdout:174 2024-09-14T03:53:55.622 INFO:tasks.workunit.client.0.smithi110.stdout:175 2024-09-14T03:53:55.623 INFO:tasks.workunit.client.0.smithi110.stdout:176 2024-09-14T03:53:55.624 INFO:tasks.workunit.client.0.smithi110.stdout:177 2024-09-14T03:53:55.624 INFO:tasks.workunit.client.0.smithi110.stdout:178 2024-09-14T03:53:55.625 INFO:tasks.workunit.client.0.smithi110.stdout:179 2024-09-14T03:53:55.626 INFO:tasks.workunit.client.0.smithi110.stdout:180 2024-09-14T03:53:55.627 INFO:tasks.workunit.client.0.smithi110.stdout:181 2024-09-14T03:53:55.628 INFO:tasks.workunit.client.0.smithi110.stdout:182 2024-09-14T03:53:55.628 INFO:tasks.workunit.client.0.smithi110.stdout:183 2024-09-14T03:53:55.629 INFO:tasks.workunit.client.0.smithi110.stdout:184 2024-09-14T03:53:55.630 INFO:tasks.workunit.client.0.smithi110.stdout:185 2024-09-14T03:53:55.631 INFO:tasks.workunit.client.0.smithi110.stdout:186 2024-09-14T03:53:55.632 INFO:tasks.workunit.client.0.smithi110.stdout:187 2024-09-14T03:53:55.632 INFO:tasks.workunit.client.0.smithi110.stdout:188 2024-09-14T03:53:55.633 INFO:tasks.workunit.client.0.smithi110.stdout:189 2024-09-14T03:53:55.634 INFO:tasks.workunit.client.0.smithi110.stdout:190 2024-09-14T03:53:55.635 INFO:tasks.workunit.client.0.smithi110.stdout:191 2024-09-14T03:53:55.635 INFO:tasks.workunit.client.0.smithi110.stdout:192 2024-09-14T03:53:55.636 INFO:tasks.workunit.client.0.smithi110.stdout:193 2024-09-14T03:53:55.637 INFO:tasks.workunit.client.0.smithi110.stdout:194 2024-09-14T03:53:55.638 INFO:tasks.workunit.client.0.smithi110.stdout:195 2024-09-14T03:53:55.638 INFO:tasks.workunit.client.0.smithi110.stdout:196 2024-09-14T03:53:55.639 INFO:tasks.workunit.client.0.smithi110.stdout:197 2024-09-14T03:53:55.640 INFO:tasks.workunit.client.0.smithi110.stdout:198 2024-09-14T03:53:55.641 INFO:tasks.workunit.client.0.smithi110.stdout:199 2024-09-14T03:53:55.641 INFO:tasks.workunit.client.0.smithi110.stdout:200 2024-09-14T03:53:55.642 INFO:tasks.workunit.client.0.smithi110.stdout:201 2024-09-14T03:53:55.643 INFO:tasks.workunit.client.0.smithi110.stdout:202 2024-09-14T03:53:55.644 INFO:tasks.workunit.client.0.smithi110.stdout:203 2024-09-14T03:53:55.645 INFO:tasks.workunit.client.0.smithi110.stdout:204 2024-09-14T03:53:55.645 INFO:tasks.workunit.client.0.smithi110.stdout:205 2024-09-14T03:53:55.646 INFO:tasks.workunit.client.0.smithi110.stdout:206 2024-09-14T03:53:55.647 INFO:tasks.workunit.client.0.smithi110.stdout:207 2024-09-14T03:53:55.648 INFO:tasks.workunit.client.0.smithi110.stdout:208 2024-09-14T03:53:55.649 INFO:tasks.workunit.client.0.smithi110.stdout:209 2024-09-14T03:53:55.649 INFO:tasks.workunit.client.0.smithi110.stdout:210 2024-09-14T03:53:55.650 INFO:tasks.workunit.client.0.smithi110.stdout:211 2024-09-14T03:53:55.651 INFO:tasks.workunit.client.0.smithi110.stdout:212 2024-09-14T03:53:55.652 INFO:tasks.workunit.client.0.smithi110.stdout:213 2024-09-14T03:53:55.652 INFO:tasks.workunit.client.0.smithi110.stdout:214 2024-09-14T03:53:55.653 INFO:tasks.workunit.client.0.smithi110.stdout:215 2024-09-14T03:53:55.654 INFO:tasks.workunit.client.0.smithi110.stdout:216 2024-09-14T03:53:55.655 INFO:tasks.workunit.client.0.smithi110.stdout:217 2024-09-14T03:53:55.656 INFO:tasks.workunit.client.0.smithi110.stdout:218 2024-09-14T03:53:55.656 INFO:tasks.workunit.client.0.smithi110.stdout:219 2024-09-14T03:53:55.657 INFO:tasks.workunit.client.0.smithi110.stdout:220 2024-09-14T03:53:55.658 INFO:tasks.workunit.client.0.smithi110.stdout:221 2024-09-14T03:53:55.659 INFO:tasks.workunit.client.0.smithi110.stdout:222 2024-09-14T03:53:55.660 INFO:tasks.workunit.client.0.smithi110.stdout:223 2024-09-14T03:53:55.661 INFO:tasks.workunit.client.0.smithi110.stdout:224 2024-09-14T03:53:55.661 INFO:tasks.workunit.client.0.smithi110.stdout:225 2024-09-14T03:53:55.662 INFO:tasks.workunit.client.0.smithi110.stdout:226 2024-09-14T03:53:55.663 INFO:tasks.workunit.client.0.smithi110.stdout:227 2024-09-14T03:53:55.664 INFO:tasks.workunit.client.0.smithi110.stdout:228 2024-09-14T03:53:55.665 INFO:tasks.workunit.client.0.smithi110.stdout:229 2024-09-14T03:53:55.665 INFO:tasks.workunit.client.0.smithi110.stdout:230 2024-09-14T03:53:55.666 INFO:tasks.workunit.client.0.smithi110.stdout:231 2024-09-14T03:53:55.667 INFO:tasks.workunit.client.0.smithi110.stdout:232 2024-09-14T03:53:55.668 INFO:tasks.workunit.client.0.smithi110.stdout:233 2024-09-14T03:53:55.669 INFO:tasks.workunit.client.0.smithi110.stdout:234 2024-09-14T03:53:55.670 INFO:tasks.workunit.client.0.smithi110.stdout:235 2024-09-14T03:53:55.670 INFO:tasks.workunit.client.0.smithi110.stdout:236 2024-09-14T03:53:55.671 INFO:tasks.workunit.client.0.smithi110.stdout:237 2024-09-14T03:53:55.672 INFO:tasks.workunit.client.0.smithi110.stdout:238 2024-09-14T03:53:55.673 INFO:tasks.workunit.client.0.smithi110.stdout:239 2024-09-14T03:53:55.673 INFO:tasks.workunit.client.0.smithi110.stdout:240 2024-09-14T03:53:55.674 INFO:tasks.workunit.client.0.smithi110.stdout:241 2024-09-14T03:53:55.675 INFO:tasks.workunit.client.0.smithi110.stdout:242 2024-09-14T03:53:55.676 INFO:tasks.workunit.client.0.smithi110.stdout:243 2024-09-14T03:53:55.677 INFO:tasks.workunit.client.0.smithi110.stdout:244 2024-09-14T03:53:55.677 INFO:tasks.workunit.client.0.smithi110.stdout:245 2024-09-14T03:53:55.678 INFO:tasks.workunit.client.0.smithi110.stdout:246 2024-09-14T03:53:55.679 INFO:tasks.workunit.client.0.smithi110.stdout:247 2024-09-14T03:53:55.680 INFO:tasks.workunit.client.0.smithi110.stdout:248 2024-09-14T03:53:55.681 INFO:tasks.workunit.client.0.smithi110.stdout:249 2024-09-14T03:53:55.681 INFO:tasks.workunit.client.0.smithi110.stdout:250 2024-09-14T03:53:55.682 INFO:tasks.workunit.client.0.smithi110.stdout:251 2024-09-14T03:53:55.683 INFO:tasks.workunit.client.0.smithi110.stdout:252 2024-09-14T03:53:55.684 INFO:tasks.workunit.client.0.smithi110.stdout:253 2024-09-14T03:53:55.685 INFO:tasks.workunit.client.0.smithi110.stdout:254 2024-09-14T03:53:55.686 INFO:tasks.workunit.client.0.smithi110.stdout:255 2024-09-14T03:53:55.686 INFO:tasks.workunit.client.0.smithi110.stdout:256 2024-09-14T03:53:55.687 INFO:tasks.workunit.client.0.smithi110.stdout:257 2024-09-14T03:53:55.688 INFO:tasks.workunit.client.0.smithi110.stdout:258 2024-09-14T03:53:55.689 INFO:tasks.workunit.client.0.smithi110.stdout:259 2024-09-14T03:53:55.690 INFO:tasks.workunit.client.0.smithi110.stdout:260 2024-09-14T03:53:55.691 INFO:tasks.workunit.client.0.smithi110.stdout:261 2024-09-14T03:53:55.691 INFO:tasks.workunit.client.0.smithi110.stdout:262 2024-09-14T03:53:55.692 INFO:tasks.workunit.client.0.smithi110.stdout:263 2024-09-14T03:53:55.693 INFO:tasks.workunit.client.0.smithi110.stdout:264 2024-09-14T03:53:55.694 INFO:tasks.workunit.client.0.smithi110.stdout:265 2024-09-14T03:53:55.695 INFO:tasks.workunit.client.0.smithi110.stdout:266 2024-09-14T03:53:55.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[88262]: pgmap v1120: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:55.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:55.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:53:55.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[88262]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T03:53:55.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1621203528' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:55.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:55.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[86423]: pgmap v1120: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:55.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:55.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:53:55.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[86423]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T03:53:55.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1621203528' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:55.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:55 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:55.697 INFO:tasks.workunit.client.0.smithi110.stdout:267 2024-09-14T03:53:55.697 INFO:tasks.workunit.client.0.smithi110.stdout:268 2024-09-14T03:53:55.698 INFO:tasks.workunit.client.0.smithi110.stdout:269 2024-09-14T03:53:55.699 INFO:tasks.workunit.client.0.smithi110.stdout:270 2024-09-14T03:53:55.700 INFO:tasks.workunit.client.0.smithi110.stdout:271 2024-09-14T03:53:55.701 INFO:tasks.workunit.client.0.smithi110.stdout:272 2024-09-14T03:53:55.702 INFO:tasks.workunit.client.0.smithi110.stdout:273 2024-09-14T03:53:55.702 INFO:tasks.workunit.client.0.smithi110.stdout:274 2024-09-14T03:53:55.703 INFO:tasks.workunit.client.0.smithi110.stdout:275 2024-09-14T03:53:55.704 INFO:tasks.workunit.client.0.smithi110.stdout:276 2024-09-14T03:53:55.705 INFO:tasks.workunit.client.0.smithi110.stdout:277 2024-09-14T03:53:55.706 INFO:tasks.workunit.client.0.smithi110.stdout:278 2024-09-14T03:53:55.707 INFO:tasks.workunit.client.0.smithi110.stdout:279 2024-09-14T03:53:55.707 INFO:tasks.workunit.client.0.smithi110.stdout:280 2024-09-14T03:53:55.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:55 smithi195 ceph-mon[82388]: pgmap v1120: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:55.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:55 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:55.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:55 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:53:55.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:55 smithi195 ceph-mon[82388]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T03:53:55.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:55 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1621203528' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:55.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:55 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:55.709 INFO:tasks.workunit.client.0.smithi110.stdout:281 2024-09-14T03:53:55.709 INFO:tasks.workunit.client.0.smithi110.stdout:282 2024-09-14T03:53:55.710 INFO:tasks.workunit.client.0.smithi110.stdout:283 2024-09-14T03:53:55.711 INFO:tasks.workunit.client.0.smithi110.stdout:284 2024-09-14T03:53:55.712 INFO:tasks.workunit.client.0.smithi110.stdout:285 2024-09-14T03:53:55.713 INFO:tasks.workunit.client.0.smithi110.stdout:286 2024-09-14T03:53:55.714 INFO:tasks.workunit.client.0.smithi110.stdout:287 2024-09-14T03:53:55.715 INFO:tasks.workunit.client.0.smithi110.stdout:288 2024-09-14T03:53:55.716 INFO:tasks.workunit.client.0.smithi110.stdout:289 2024-09-14T03:53:55.716 INFO:tasks.workunit.client.0.smithi110.stdout:290 2024-09-14T03:53:55.717 INFO:tasks.workunit.client.0.smithi110.stdout:291 2024-09-14T03:53:55.718 INFO:tasks.workunit.client.0.smithi110.stdout:292 2024-09-14T03:53:55.719 INFO:tasks.workunit.client.0.smithi110.stdout:293 2024-09-14T03:53:55.720 INFO:tasks.workunit.client.0.smithi110.stdout:294 2024-09-14T03:53:55.720 INFO:tasks.workunit.client.0.smithi110.stdout:295 2024-09-14T03:53:55.721 INFO:tasks.workunit.client.0.smithi110.stdout:296 2024-09-14T03:53:55.722 INFO:tasks.workunit.client.0.smithi110.stdout:297 2024-09-14T03:53:55.723 INFO:tasks.workunit.client.0.smithi110.stdout:298 2024-09-14T03:53:55.724 INFO:tasks.workunit.client.0.smithi110.stdout:299 2024-09-14T03:53:55.725 INFO:tasks.workunit.client.0.smithi110.stdout:300 2024-09-14T03:53:55.726 INFO:tasks.workunit.client.0.smithi110.stdout:301 2024-09-14T03:53:55.727 INFO:tasks.workunit.client.0.smithi110.stdout:302 2024-09-14T03:53:55.728 INFO:tasks.workunit.client.0.smithi110.stdout:303 2024-09-14T03:53:55.729 INFO:tasks.workunit.client.0.smithi110.stdout:304 2024-09-14T03:53:55.730 INFO:tasks.workunit.client.0.smithi110.stdout:305 2024-09-14T03:53:55.731 INFO:tasks.workunit.client.0.smithi110.stdout:306 2024-09-14T03:53:55.732 INFO:tasks.workunit.client.0.smithi110.stdout:307 2024-09-14T03:53:55.733 INFO:tasks.workunit.client.0.smithi110.stdout:308 2024-09-14T03:53:55.734 INFO:tasks.workunit.client.0.smithi110.stdout:309 2024-09-14T03:53:55.735 INFO:tasks.workunit.client.0.smithi110.stdout:310 2024-09-14T03:53:55.736 INFO:tasks.workunit.client.0.smithi110.stdout:311 2024-09-14T03:53:55.737 INFO:tasks.workunit.client.0.smithi110.stdout:312 2024-09-14T03:53:55.738 INFO:tasks.workunit.client.0.smithi110.stdout:313 2024-09-14T03:53:55.739 INFO:tasks.workunit.client.0.smithi110.stdout:314 2024-09-14T03:53:55.740 INFO:tasks.workunit.client.0.smithi110.stdout:315 2024-09-14T03:53:55.740 INFO:tasks.workunit.client.0.smithi110.stdout:316 2024-09-14T03:53:55.742 INFO:tasks.workunit.client.0.smithi110.stdout:317 2024-09-14T03:53:55.743 INFO:tasks.workunit.client.0.smithi110.stdout:318 2024-09-14T03:53:55.744 INFO:tasks.workunit.client.0.smithi110.stdout:319 2024-09-14T03:53:55.745 INFO:tasks.workunit.client.0.smithi110.stdout:320 2024-09-14T03:53:55.745 INFO:tasks.workunit.client.0.smithi110.stdout:321 2024-09-14T03:53:55.746 INFO:tasks.workunit.client.0.smithi110.stdout:322 2024-09-14T03:53:55.747 INFO:tasks.workunit.client.0.smithi110.stdout:323 2024-09-14T03:53:55.748 INFO:tasks.workunit.client.0.smithi110.stdout:324 2024-09-14T03:53:55.749 INFO:tasks.workunit.client.0.smithi110.stdout:325 2024-09-14T03:53:55.750 INFO:tasks.workunit.client.0.smithi110.stdout:326 2024-09-14T03:53:55.751 INFO:tasks.workunit.client.0.smithi110.stdout:327 2024-09-14T03:53:55.752 INFO:tasks.workunit.client.0.smithi110.stdout:328 2024-09-14T03:53:55.753 INFO:tasks.workunit.client.0.smithi110.stdout:329 2024-09-14T03:53:55.754 INFO:tasks.workunit.client.0.smithi110.stdout:330 2024-09-14T03:53:55.756 INFO:tasks.workunit.client.0.smithi110.stdout:331 2024-09-14T03:53:55.757 INFO:tasks.workunit.client.0.smithi110.stdout:332 2024-09-14T03:53:55.758 INFO:tasks.workunit.client.0.smithi110.stdout:333 2024-09-14T03:53:55.758 INFO:tasks.workunit.client.0.smithi110.stdout:334 2024-09-14T03:53:55.759 INFO:tasks.workunit.client.0.smithi110.stdout:335 2024-09-14T03:53:55.760 INFO:tasks.workunit.client.0.smithi110.stdout:336 2024-09-14T03:53:55.761 INFO:tasks.workunit.client.0.smithi110.stdout:337 2024-09-14T03:53:55.762 INFO:tasks.workunit.client.0.smithi110.stdout:338 2024-09-14T03:53:55.762 INFO:tasks.workunit.client.0.smithi110.stdout:339 2024-09-14T03:53:55.763 INFO:tasks.workunit.client.0.smithi110.stdout:340 2024-09-14T03:53:55.764 INFO:tasks.workunit.client.0.smithi110.stdout:341 2024-09-14T03:53:55.765 INFO:tasks.workunit.client.0.smithi110.stdout:342 2024-09-14T03:53:55.766 INFO:tasks.workunit.client.0.smithi110.stdout:343 2024-09-14T03:53:55.767 INFO:tasks.workunit.client.0.smithi110.stdout:344 2024-09-14T03:53:55.768 INFO:tasks.workunit.client.0.smithi110.stdout:345 2024-09-14T03:53:55.769 INFO:tasks.workunit.client.0.smithi110.stdout:346 2024-09-14T03:53:55.770 INFO:tasks.workunit.client.0.smithi110.stdout:347 2024-09-14T03:53:55.771 INFO:tasks.workunit.client.0.smithi110.stdout:348 2024-09-14T03:53:55.772 INFO:tasks.workunit.client.0.smithi110.stdout:349 2024-09-14T03:53:55.773 INFO:tasks.workunit.client.0.smithi110.stdout:350 2024-09-14T03:53:55.774 INFO:tasks.workunit.client.0.smithi110.stdout:351 2024-09-14T03:53:55.775 INFO:tasks.workunit.client.0.smithi110.stdout:352 2024-09-14T03:53:55.776 INFO:tasks.workunit.client.0.smithi110.stdout:353 2024-09-14T03:53:55.777 INFO:tasks.workunit.client.0.smithi110.stdout:354 2024-09-14T03:53:55.778 INFO:tasks.workunit.client.0.smithi110.stdout:355 2024-09-14T03:53:55.779 INFO:tasks.workunit.client.0.smithi110.stdout:356 2024-09-14T03:53:55.780 INFO:tasks.workunit.client.0.smithi110.stdout:357 2024-09-14T03:53:55.781 INFO:tasks.workunit.client.0.smithi110.stdout:358 2024-09-14T03:53:55.783 INFO:tasks.workunit.client.0.smithi110.stdout:359 2024-09-14T03:53:55.784 INFO:tasks.workunit.client.0.smithi110.stdout:360 2024-09-14T03:53:55.785 INFO:tasks.workunit.client.0.smithi110.stdout:361 2024-09-14T03:53:55.786 INFO:tasks.workunit.client.0.smithi110.stdout:362 2024-09-14T03:53:55.787 INFO:tasks.workunit.client.0.smithi110.stdout:363 2024-09-14T03:53:55.788 INFO:tasks.workunit.client.0.smithi110.stdout:364 2024-09-14T03:53:55.789 INFO:tasks.workunit.client.0.smithi110.stdout:365 2024-09-14T03:53:55.790 INFO:tasks.workunit.client.0.smithi110.stdout:366 2024-09-14T03:53:55.791 INFO:tasks.workunit.client.0.smithi110.stdout:367 2024-09-14T03:53:55.792 INFO:tasks.workunit.client.0.smithi110.stdout:368 2024-09-14T03:53:55.792 INFO:tasks.workunit.client.0.smithi110.stdout:369 2024-09-14T03:53:55.793 INFO:tasks.workunit.client.0.smithi110.stdout:370 2024-09-14T03:53:55.794 INFO:tasks.workunit.client.0.smithi110.stdout:371 2024-09-14T03:53:55.795 INFO:tasks.workunit.client.0.smithi110.stdout:372 2024-09-14T03:53:55.796 INFO:tasks.workunit.client.0.smithi110.stdout:373 2024-09-14T03:53:55.798 INFO:tasks.workunit.client.0.smithi110.stdout:374 2024-09-14T03:53:55.799 INFO:tasks.workunit.client.0.smithi110.stdout:375 2024-09-14T03:53:55.800 INFO:tasks.workunit.client.0.smithi110.stdout:376 2024-09-14T03:53:55.801 INFO:tasks.workunit.client.0.smithi110.stdout:377 2024-09-14T03:53:55.802 INFO:tasks.workunit.client.0.smithi110.stdout:378 2024-09-14T03:53:55.803 INFO:tasks.workunit.client.0.smithi110.stdout:379 2024-09-14T03:53:55.804 INFO:tasks.workunit.client.0.smithi110.stdout:380 2024-09-14T03:53:55.805 INFO:tasks.workunit.client.0.smithi110.stdout:381 2024-09-14T03:53:55.806 INFO:tasks.workunit.client.0.smithi110.stdout:382 2024-09-14T03:53:55.807 INFO:tasks.workunit.client.0.smithi110.stdout:383 2024-09-14T03:53:55.808 INFO:tasks.workunit.client.0.smithi110.stdout:384 2024-09-14T03:53:55.809 INFO:tasks.workunit.client.0.smithi110.stdout:385 2024-09-14T03:53:55.810 INFO:tasks.workunit.client.0.smithi110.stdout:386 2024-09-14T03:53:55.811 INFO:tasks.workunit.client.0.smithi110.stdout:387 2024-09-14T03:53:55.812 INFO:tasks.workunit.client.0.smithi110.stdout:388 2024-09-14T03:53:55.812 INFO:tasks.workunit.client.0.smithi110.stdout:389 2024-09-14T03:53:55.813 INFO:tasks.workunit.client.0.smithi110.stdout:390 2024-09-14T03:53:55.814 INFO:tasks.workunit.client.0.smithi110.stdout:391 2024-09-14T03:53:55.815 INFO:tasks.workunit.client.0.smithi110.stdout:392 2024-09-14T03:53:55.816 INFO:tasks.workunit.client.0.smithi110.stdout:393 2024-09-14T03:53:55.817 INFO:tasks.workunit.client.0.smithi110.stdout:394 2024-09-14T03:53:55.818 INFO:tasks.workunit.client.0.smithi110.stdout:395 2024-09-14T03:53:55.818 INFO:tasks.workunit.client.0.smithi110.stdout:396 2024-09-14T03:53:55.819 INFO:tasks.workunit.client.0.smithi110.stdout:397 2024-09-14T03:53:55.820 INFO:tasks.workunit.client.0.smithi110.stdout:398 2024-09-14T03:53:55.821 INFO:tasks.workunit.client.0.smithi110.stdout:399 2024-09-14T03:53:55.822 INFO:tasks.workunit.client.0.smithi110.stdout:400 2024-09-14T03:53:55.823 INFO:tasks.workunit.client.0.smithi110.stdout:401 2024-09-14T03:53:55.824 INFO:tasks.workunit.client.0.smithi110.stdout:402 2024-09-14T03:53:55.825 INFO:tasks.workunit.client.0.smithi110.stdout:403 2024-09-14T03:53:55.826 INFO:tasks.workunit.client.0.smithi110.stdout:404 2024-09-14T03:53:55.826 INFO:tasks.workunit.client.0.smithi110.stdout:405 2024-09-14T03:53:55.827 INFO:tasks.workunit.client.0.smithi110.stdout:406 2024-09-14T03:53:55.828 INFO:tasks.workunit.client.0.smithi110.stdout:407 2024-09-14T03:53:55.829 INFO:tasks.workunit.client.0.smithi110.stdout:408 2024-09-14T03:53:55.830 INFO:tasks.workunit.client.0.smithi110.stdout:409 2024-09-14T03:53:55.830 INFO:tasks.workunit.client.0.smithi110.stdout:410 2024-09-14T03:53:55.832 INFO:tasks.workunit.client.0.smithi110.stdout:411 2024-09-14T03:53:55.832 INFO:tasks.workunit.client.0.smithi110.stdout:412 2024-09-14T03:53:55.833 INFO:tasks.workunit.client.0.smithi110.stdout:413 2024-09-14T03:53:55.834 INFO:tasks.workunit.client.0.smithi110.stdout:414 2024-09-14T03:53:55.835 INFO:tasks.workunit.client.0.smithi110.stdout:415 2024-09-14T03:53:55.836 INFO:tasks.workunit.client.0.smithi110.stdout:416 2024-09-14T03:53:55.837 INFO:tasks.workunit.client.0.smithi110.stdout:417 2024-09-14T03:53:55.838 INFO:tasks.workunit.client.0.smithi110.stdout:418 2024-09-14T03:53:55.839 INFO:tasks.workunit.client.0.smithi110.stdout:419 2024-09-14T03:53:55.839 INFO:tasks.workunit.client.0.smithi110.stdout:420 2024-09-14T03:53:55.840 INFO:tasks.workunit.client.0.smithi110.stdout:421 2024-09-14T03:53:55.841 INFO:tasks.workunit.client.0.smithi110.stdout:422 2024-09-14T03:53:55.842 INFO:tasks.workunit.client.0.smithi110.stdout:423 2024-09-14T03:53:55.843 INFO:tasks.workunit.client.0.smithi110.stdout:424 2024-09-14T03:53:55.844 INFO:tasks.workunit.client.0.smithi110.stdout:425 2024-09-14T03:53:55.845 INFO:tasks.workunit.client.0.smithi110.stdout:426 2024-09-14T03:53:55.846 INFO:tasks.workunit.client.0.smithi110.stdout:427 2024-09-14T03:53:55.846 INFO:tasks.workunit.client.0.smithi110.stdout:428 2024-09-14T03:53:55.847 INFO:tasks.workunit.client.0.smithi110.stdout:429 2024-09-14T03:53:55.848 INFO:tasks.workunit.client.0.smithi110.stdout:430 2024-09-14T03:53:55.849 INFO:tasks.workunit.client.0.smithi110.stdout:431 2024-09-14T03:53:55.850 INFO:tasks.workunit.client.0.smithi110.stdout:432 2024-09-14T03:53:55.851 INFO:tasks.workunit.client.0.smithi110.stdout:433 2024-09-14T03:53:55.852 INFO:tasks.workunit.client.0.smithi110.stdout:434 2024-09-14T03:53:55.853 INFO:tasks.workunit.client.0.smithi110.stdout:435 2024-09-14T03:53:55.853 INFO:tasks.workunit.client.0.smithi110.stdout:436 2024-09-14T03:53:55.854 INFO:tasks.workunit.client.0.smithi110.stdout:437 2024-09-14T03:53:55.855 INFO:tasks.workunit.client.0.smithi110.stdout:438 2024-09-14T03:53:55.856 INFO:tasks.workunit.client.0.smithi110.stdout:439 2024-09-14T03:53:55.857 INFO:tasks.workunit.client.0.smithi110.stdout:440 2024-09-14T03:53:55.858 INFO:tasks.workunit.client.0.smithi110.stdout:441 2024-09-14T03:53:55.859 INFO:tasks.workunit.client.0.smithi110.stdout:442 2024-09-14T03:53:55.860 INFO:tasks.workunit.client.0.smithi110.stdout:443 2024-09-14T03:53:55.861 INFO:tasks.workunit.client.0.smithi110.stdout:444 2024-09-14T03:53:55.861 INFO:tasks.workunit.client.0.smithi110.stdout:445 2024-09-14T03:53:55.862 INFO:tasks.workunit.client.0.smithi110.stdout:446 2024-09-14T03:53:55.864 INFO:tasks.workunit.client.0.smithi110.stdout:447 2024-09-14T03:53:55.864 INFO:tasks.workunit.client.0.smithi110.stdout:448 2024-09-14T03:53:55.865 INFO:tasks.workunit.client.0.smithi110.stdout:449 2024-09-14T03:53:55.866 INFO:tasks.workunit.client.0.smithi110.stdout:450 2024-09-14T03:53:55.867 INFO:tasks.workunit.client.0.smithi110.stdout:451 2024-09-14T03:53:55.867 INFO:tasks.workunit.client.0.smithi110.stdout:452 2024-09-14T03:53:55.868 INFO:tasks.workunit.client.0.smithi110.stdout:453 2024-09-14T03:53:55.869 INFO:tasks.workunit.client.0.smithi110.stdout:454 2024-09-14T03:53:55.870 INFO:tasks.workunit.client.0.smithi110.stdout:455 2024-09-14T03:53:55.871 INFO:tasks.workunit.client.0.smithi110.stdout:456 2024-09-14T03:53:55.872 INFO:tasks.workunit.client.0.smithi110.stdout:457 2024-09-14T03:53:55.873 INFO:tasks.workunit.client.0.smithi110.stdout:458 2024-09-14T03:53:55.873 INFO:tasks.workunit.client.0.smithi110.stdout:459 2024-09-14T03:53:55.874 INFO:tasks.workunit.client.0.smithi110.stdout:460 2024-09-14T03:53:55.875 INFO:tasks.workunit.client.0.smithi110.stdout:461 2024-09-14T03:53:55.876 INFO:tasks.workunit.client.0.smithi110.stdout:462 2024-09-14T03:53:55.877 INFO:tasks.workunit.client.0.smithi110.stdout:463 2024-09-14T03:53:55.878 INFO:tasks.workunit.client.0.smithi110.stdout:464 2024-09-14T03:53:55.879 INFO:tasks.workunit.client.0.smithi110.stdout:465 2024-09-14T03:53:55.879 INFO:tasks.workunit.client.0.smithi110.stdout:466 2024-09-14T03:53:55.880 INFO:tasks.workunit.client.0.smithi110.stdout:467 2024-09-14T03:53:55.881 INFO:tasks.workunit.client.0.smithi110.stdout:468 2024-09-14T03:53:55.882 INFO:tasks.workunit.client.0.smithi110.stdout:469 2024-09-14T03:53:55.883 INFO:tasks.workunit.client.0.smithi110.stdout:470 2024-09-14T03:53:55.884 INFO:tasks.workunit.client.0.smithi110.stdout:471 2024-09-14T03:53:55.885 INFO:tasks.workunit.client.0.smithi110.stdout:472 2024-09-14T03:53:55.886 INFO:tasks.workunit.client.0.smithi110.stdout:473 2024-09-14T03:53:55.886 INFO:tasks.workunit.client.0.smithi110.stdout:474 2024-09-14T03:53:55.887 INFO:tasks.workunit.client.0.smithi110.stdout:475 2024-09-14T03:53:55.888 INFO:tasks.workunit.client.0.smithi110.stdout:476 2024-09-14T03:53:55.893 INFO:tasks.workunit.client.0.smithi110.stdout:477 2024-09-14T03:53:55.893 INFO:tasks.workunit.client.0.smithi110.stdout:478 2024-09-14T03:53:55.893 INFO:tasks.workunit.client.0.smithi110.stdout:479 2024-09-14T03:53:55.893 INFO:tasks.workunit.client.0.smithi110.stdout:480 2024-09-14T03:53:55.894 INFO:tasks.workunit.client.0.smithi110.stdout:481 2024-09-14T03:53:55.894 INFO:tasks.workunit.client.0.smithi110.stdout:482 2024-09-14T03:53:55.894 INFO:tasks.workunit.client.0.smithi110.stdout:483 2024-09-14T03:53:55.895 INFO:tasks.workunit.client.0.smithi110.stdout:484 2024-09-14T03:53:55.896 INFO:tasks.workunit.client.0.smithi110.stdout:485 2024-09-14T03:53:55.897 INFO:tasks.workunit.client.0.smithi110.stdout:486 2024-09-14T03:53:55.898 INFO:tasks.workunit.client.0.smithi110.stdout:487 2024-09-14T03:53:55.899 INFO:tasks.workunit.client.0.smithi110.stdout:488 2024-09-14T03:53:55.900 INFO:tasks.workunit.client.0.smithi110.stdout:489 2024-09-14T03:53:55.901 INFO:tasks.workunit.client.0.smithi110.stdout:490 2024-09-14T03:53:55.902 INFO:tasks.workunit.client.0.smithi110.stdout:491 2024-09-14T03:53:55.903 INFO:tasks.workunit.client.0.smithi110.stdout:492 2024-09-14T03:53:55.904 INFO:tasks.workunit.client.0.smithi110.stdout:493 2024-09-14T03:53:55.905 INFO:tasks.workunit.client.0.smithi110.stdout:494 2024-09-14T03:53:55.906 INFO:tasks.workunit.client.0.smithi110.stdout:495 2024-09-14T03:53:55.907 INFO:tasks.workunit.client.0.smithi110.stdout:496 2024-09-14T03:53:55.908 INFO:tasks.workunit.client.0.smithi110.stdout:497 2024-09-14T03:53:55.909 INFO:tasks.workunit.client.0.smithi110.stdout:498 2024-09-14T03:53:55.909 INFO:tasks.workunit.client.0.smithi110.stdout:499 2024-09-14T03:53:55.910 INFO:tasks.workunit.client.0.smithi110.stdout:500 2024-09-14T03:53:55.911 INFO:tasks.workunit.client.0.smithi110.stdout:501 2024-09-14T03:53:55.912 INFO:tasks.workunit.client.0.smithi110.stdout:502 2024-09-14T03:53:55.913 INFO:tasks.workunit.client.0.smithi110.stdout:503 2024-09-14T03:53:55.914 INFO:tasks.workunit.client.0.smithi110.stdout:504 2024-09-14T03:53:55.915 INFO:tasks.workunit.client.0.smithi110.stdout:505 2024-09-14T03:53:55.916 INFO:tasks.workunit.client.0.smithi110.stdout:506 2024-09-14T03:53:55.917 INFO:tasks.workunit.client.0.smithi110.stdout:507 2024-09-14T03:53:55.918 INFO:tasks.workunit.client.0.smithi110.stdout:508 2024-09-14T03:53:55.919 INFO:tasks.workunit.client.0.smithi110.stdout:509 2024-09-14T03:53:55.920 INFO:tasks.workunit.client.0.smithi110.stdout:510 2024-09-14T03:53:55.921 INFO:tasks.workunit.client.0.smithi110.stdout:511 2024-09-14T03:53:55.922 INFO:tasks.workunit.client.0.smithi110.stdout:512 2024-09-14T03:53:55.924 INFO:tasks.workunit.client.0.smithi110.stdout:513 2024-09-14T03:53:55.924 INFO:tasks.workunit.client.0.smithi110.stdout:514 2024-09-14T03:53:55.925 INFO:tasks.workunit.client.0.smithi110.stdout:515 2024-09-14T03:53:55.926 INFO:tasks.workunit.client.0.smithi110.stdout:516 2024-09-14T03:53:55.927 INFO:tasks.workunit.client.0.smithi110.stdout:517 2024-09-14T03:53:55.928 INFO:tasks.workunit.client.0.smithi110.stdout:518 2024-09-14T03:53:55.929 INFO:tasks.workunit.client.0.smithi110.stdout:519 2024-09-14T03:53:55.930 INFO:tasks.workunit.client.0.smithi110.stdout:520 2024-09-14T03:53:55.931 INFO:tasks.workunit.client.0.smithi110.stdout:521 2024-09-14T03:53:55.931 INFO:tasks.workunit.client.0.smithi110.stdout:522 2024-09-14T03:53:55.932 INFO:tasks.workunit.client.0.smithi110.stdout:523 2024-09-14T03:53:55.934 INFO:tasks.workunit.client.0.smithi110.stdout:524 2024-09-14T03:53:55.934 INFO:tasks.workunit.client.0.smithi110.stdout:525 2024-09-14T03:53:55.935 INFO:tasks.workunit.client.0.smithi110.stdout:526 2024-09-14T03:53:55.936 INFO:tasks.workunit.client.0.smithi110.stdout:527 2024-09-14T03:53:55.937 INFO:tasks.workunit.client.0.smithi110.stdout:528 2024-09-14T03:53:55.938 INFO:tasks.workunit.client.0.smithi110.stdout:529 2024-09-14T03:53:55.939 INFO:tasks.workunit.client.0.smithi110.stdout:530 2024-09-14T03:53:55.940 INFO:tasks.workunit.client.0.smithi110.stdout:531 2024-09-14T03:53:55.941 INFO:tasks.workunit.client.0.smithi110.stdout:532 2024-09-14T03:53:55.942 INFO:tasks.workunit.client.0.smithi110.stdout:533 2024-09-14T03:53:55.943 INFO:tasks.workunit.client.0.smithi110.stdout:534 2024-09-14T03:53:55.943 INFO:tasks.workunit.client.0.smithi110.stdout:535 2024-09-14T03:53:55.944 INFO:tasks.workunit.client.0.smithi110.stdout:536 2024-09-14T03:53:55.945 INFO:tasks.workunit.client.0.smithi110.stdout:537 2024-09-14T03:53:55.946 INFO:tasks.workunit.client.0.smithi110.stdout:538 2024-09-14T03:53:55.947 INFO:tasks.workunit.client.0.smithi110.stdout:539 2024-09-14T03:53:55.949 INFO:tasks.workunit.client.0.smithi110.stdout:540 2024-09-14T03:53:55.949 INFO:tasks.workunit.client.0.smithi110.stdout:541 2024-09-14T03:53:55.951 INFO:tasks.workunit.client.0.smithi110.stdout:542 2024-09-14T03:53:55.951 INFO:tasks.workunit.client.0.smithi110.stdout:543 2024-09-14T03:53:55.952 INFO:tasks.workunit.client.0.smithi110.stdout:544 2024-09-14T03:53:55.953 INFO:tasks.workunit.client.0.smithi110.stdout:545 2024-09-14T03:53:55.954 INFO:tasks.workunit.client.0.smithi110.stdout:546 2024-09-14T03:53:55.955 INFO:tasks.workunit.client.0.smithi110.stdout:547 2024-09-14T03:53:55.956 INFO:tasks.workunit.client.0.smithi110.stdout:548 2024-09-14T03:53:55.957 INFO:tasks.workunit.client.0.smithi110.stdout:549 2024-09-14T03:53:55.958 INFO:tasks.workunit.client.0.smithi110.stdout:550 2024-09-14T03:53:55.959 INFO:tasks.workunit.client.0.smithi110.stdout:551 2024-09-14T03:53:55.960 INFO:tasks.workunit.client.0.smithi110.stdout:552 2024-09-14T03:53:55.960 INFO:tasks.workunit.client.0.smithi110.stdout:553 2024-09-14T03:53:55.961 INFO:tasks.workunit.client.0.smithi110.stdout:554 2024-09-14T03:53:55.962 INFO:tasks.workunit.client.0.smithi110.stdout:555 2024-09-14T03:53:55.963 INFO:tasks.workunit.client.0.smithi110.stdout:556 2024-09-14T03:53:55.964 INFO:tasks.workunit.client.0.smithi110.stdout:557 2024-09-14T03:53:55.965 INFO:tasks.workunit.client.0.smithi110.stdout:558 2024-09-14T03:53:55.966 INFO:tasks.workunit.client.0.smithi110.stdout:559 2024-09-14T03:53:55.967 INFO:tasks.workunit.client.0.smithi110.stdout:560 2024-09-14T03:53:55.968 INFO:tasks.workunit.client.0.smithi110.stdout:561 2024-09-14T03:53:55.968 INFO:tasks.workunit.client.0.smithi110.stdout:562 2024-09-14T03:53:55.969 INFO:tasks.workunit.client.0.smithi110.stdout:563 2024-09-14T03:53:55.970 INFO:tasks.workunit.client.0.smithi110.stdout:564 2024-09-14T03:53:55.971 INFO:tasks.workunit.client.0.smithi110.stdout:565 2024-09-14T03:53:55.972 INFO:tasks.workunit.client.0.smithi110.stdout:566 2024-09-14T03:53:55.973 INFO:tasks.workunit.client.0.smithi110.stdout:567 2024-09-14T03:53:55.974 INFO:tasks.workunit.client.0.smithi110.stdout:568 2024-09-14T03:53:55.975 INFO:tasks.workunit.client.0.smithi110.stdout:569 2024-09-14T03:53:55.976 INFO:tasks.workunit.client.0.smithi110.stdout:570 2024-09-14T03:53:55.977 INFO:tasks.workunit.client.0.smithi110.stdout:571 2024-09-14T03:53:55.978 INFO:tasks.workunit.client.0.smithi110.stdout:572 2024-09-14T03:53:55.978 INFO:tasks.workunit.client.0.smithi110.stdout:573 2024-09-14T03:53:55.979 INFO:tasks.workunit.client.0.smithi110.stdout:574 2024-09-14T03:53:55.980 INFO:tasks.workunit.client.0.smithi110.stdout:575 2024-09-14T03:53:55.981 INFO:tasks.workunit.client.0.smithi110.stdout:576 2024-09-14T03:53:55.982 INFO:tasks.workunit.client.0.smithi110.stdout:577 2024-09-14T03:53:55.983 INFO:tasks.workunit.client.0.smithi110.stdout:578 2024-09-14T03:53:55.984 INFO:tasks.workunit.client.0.smithi110.stdout:579 2024-09-14T03:53:55.985 INFO:tasks.workunit.client.0.smithi110.stdout:580 2024-09-14T03:53:55.986 INFO:tasks.workunit.client.0.smithi110.stdout:581 2024-09-14T03:53:55.987 INFO:tasks.workunit.client.0.smithi110.stdout:582 2024-09-14T03:53:55.987 INFO:tasks.workunit.client.0.smithi110.stdout:583 2024-09-14T03:53:55.988 INFO:tasks.workunit.client.0.smithi110.stdout:584 2024-09-14T03:53:55.989 INFO:tasks.workunit.client.0.smithi110.stdout:585 2024-09-14T03:53:55.990 INFO:tasks.workunit.client.0.smithi110.stdout:586 2024-09-14T03:53:55.991 INFO:tasks.workunit.client.0.smithi110.stdout:587 2024-09-14T03:53:55.992 INFO:tasks.workunit.client.0.smithi110.stdout:588 2024-09-14T03:53:55.993 INFO:tasks.workunit.client.0.smithi110.stdout:589 2024-09-14T03:53:55.994 INFO:tasks.workunit.client.0.smithi110.stdout:590 2024-09-14T03:53:55.995 INFO:tasks.workunit.client.0.smithi110.stdout:591 2024-09-14T03:53:55.996 INFO:tasks.workunit.client.0.smithi110.stdout:592 2024-09-14T03:53:55.997 INFO:tasks.workunit.client.0.smithi110.stdout:593 2024-09-14T03:53:55.997 INFO:tasks.workunit.client.0.smithi110.stdout:594 2024-09-14T03:53:55.998 INFO:tasks.workunit.client.0.smithi110.stdout:595 2024-09-14T03:53:55.999 INFO:tasks.workunit.client.0.smithi110.stdout:596 2024-09-14T03:53:56.000 INFO:tasks.workunit.client.0.smithi110.stdout:597 2024-09-14T03:53:56.001 INFO:tasks.workunit.client.0.smithi110.stdout:598 2024-09-14T03:53:56.002 INFO:tasks.workunit.client.0.smithi110.stdout:599 2024-09-14T03:53:56.003 INFO:tasks.workunit.client.0.smithi110.stdout:600 2024-09-14T03:53:56.004 INFO:tasks.workunit.client.0.smithi110.stdout:601 2024-09-14T03:53:56.005 INFO:tasks.workunit.client.0.smithi110.stdout:602 2024-09-14T03:53:56.005 INFO:tasks.workunit.client.0.smithi110.stdout:603 2024-09-14T03:53:56.006 INFO:tasks.workunit.client.0.smithi110.stdout:604 2024-09-14T03:53:56.007 INFO:tasks.workunit.client.0.smithi110.stdout:605 2024-09-14T03:53:56.009 INFO:tasks.workunit.client.0.smithi110.stdout:606 2024-09-14T03:53:56.010 INFO:tasks.workunit.client.0.smithi110.stdout:607 2024-09-14T03:53:56.011 INFO:tasks.workunit.client.0.smithi110.stdout:608 2024-09-14T03:53:56.011 INFO:tasks.workunit.client.0.smithi110.stdout:609 2024-09-14T03:53:56.012 INFO:tasks.workunit.client.0.smithi110.stdout:610 2024-09-14T03:53:56.013 INFO:tasks.workunit.client.0.smithi110.stdout:611 2024-09-14T03:53:56.014 INFO:tasks.workunit.client.0.smithi110.stdout:612 2024-09-14T03:53:56.015 INFO:tasks.workunit.client.0.smithi110.stdout:613 2024-09-14T03:53:56.016 INFO:tasks.workunit.client.0.smithi110.stdout:614 2024-09-14T03:53:56.017 INFO:tasks.workunit.client.0.smithi110.stdout:615 2024-09-14T03:53:56.018 INFO:tasks.workunit.client.0.smithi110.stdout:616 2024-09-14T03:53:56.019 INFO:tasks.workunit.client.0.smithi110.stdout:617 2024-09-14T03:53:56.019 INFO:tasks.workunit.client.0.smithi110.stdout:618 2024-09-14T03:53:56.020 INFO:tasks.workunit.client.0.smithi110.stdout:619 2024-09-14T03:53:56.021 INFO:tasks.workunit.client.0.smithi110.stdout:620 2024-09-14T03:53:56.022 INFO:tasks.workunit.client.0.smithi110.stdout:621 2024-09-14T03:53:56.023 INFO:tasks.workunit.client.0.smithi110.stdout:622 2024-09-14T03:53:56.024 INFO:tasks.workunit.client.0.smithi110.stdout:623 2024-09-14T03:53:56.025 INFO:tasks.workunit.client.0.smithi110.stdout:624 2024-09-14T03:53:56.026 INFO:tasks.workunit.client.0.smithi110.stdout:625 2024-09-14T03:53:56.027 INFO:tasks.workunit.client.0.smithi110.stdout:626 2024-09-14T03:53:56.028 INFO:tasks.workunit.client.0.smithi110.stdout:627 2024-09-14T03:53:56.029 INFO:tasks.workunit.client.0.smithi110.stdout:628 2024-09-14T03:53:56.030 INFO:tasks.workunit.client.0.smithi110.stdout:629 2024-09-14T03:53:56.030 INFO:tasks.workunit.client.0.smithi110.stdout:630 2024-09-14T03:53:56.031 INFO:tasks.workunit.client.0.smithi110.stdout:631 2024-09-14T03:53:56.033 INFO:tasks.workunit.client.0.smithi110.stdout:632 2024-09-14T03:53:56.034 INFO:tasks.workunit.client.0.smithi110.stdout:633 2024-09-14T03:53:56.034 INFO:tasks.workunit.client.0.smithi110.stdout:634 2024-09-14T03:53:56.035 INFO:tasks.workunit.client.0.smithi110.stdout:635 2024-09-14T03:53:56.036 INFO:tasks.workunit.client.0.smithi110.stdout:636 2024-09-14T03:53:56.037 INFO:tasks.workunit.client.0.smithi110.stdout:637 2024-09-14T03:53:56.038 INFO:tasks.workunit.client.0.smithi110.stdout:638 2024-09-14T03:53:56.039 INFO:tasks.workunit.client.0.smithi110.stdout:639 2024-09-14T03:53:56.040 INFO:tasks.workunit.client.0.smithi110.stdout:640 2024-09-14T03:53:56.041 INFO:tasks.workunit.client.0.smithi110.stdout:641 2024-09-14T03:53:56.042 INFO:tasks.workunit.client.0.smithi110.stdout:642 2024-09-14T03:53:56.043 INFO:tasks.workunit.client.0.smithi110.stdout:643 2024-09-14T03:53:56.044 INFO:tasks.workunit.client.0.smithi110.stdout:644 2024-09-14T03:53:56.045 INFO:tasks.workunit.client.0.smithi110.stdout:645 2024-09-14T03:53:56.046 INFO:tasks.workunit.client.0.smithi110.stdout:646 2024-09-14T03:53:56.047 INFO:tasks.workunit.client.0.smithi110.stdout:647 2024-09-14T03:53:56.048 INFO:tasks.workunit.client.0.smithi110.stdout:648 2024-09-14T03:53:56.049 INFO:tasks.workunit.client.0.smithi110.stdout:649 2024-09-14T03:53:56.049 INFO:tasks.workunit.client.0.smithi110.stdout:650 2024-09-14T03:53:56.050 INFO:tasks.workunit.client.0.smithi110.stdout:651 2024-09-14T03:53:56.051 INFO:tasks.workunit.client.0.smithi110.stdout:652 2024-09-14T03:53:56.053 INFO:tasks.workunit.client.0.smithi110.stdout:653 2024-09-14T03:53:56.053 INFO:tasks.workunit.client.0.smithi110.stdout:654 2024-09-14T03:53:56.054 INFO:tasks.workunit.client.0.smithi110.stdout:655 2024-09-14T03:53:56.055 INFO:tasks.workunit.client.0.smithi110.stdout:656 2024-09-14T03:53:56.056 INFO:tasks.workunit.client.0.smithi110.stdout:657 2024-09-14T03:53:56.057 INFO:tasks.workunit.client.0.smithi110.stdout:658 2024-09-14T03:53:56.058 INFO:tasks.workunit.client.0.smithi110.stdout:659 2024-09-14T03:53:56.059 INFO:tasks.workunit.client.0.smithi110.stdout:660 2024-09-14T03:53:56.060 INFO:tasks.workunit.client.0.smithi110.stdout:661 2024-09-14T03:53:56.061 INFO:tasks.workunit.client.0.smithi110.stdout:662 2024-09-14T03:53:56.063 INFO:tasks.workunit.client.0.smithi110.stdout:663 2024-09-14T03:53:56.063 INFO:tasks.workunit.client.0.smithi110.stdout:664 2024-09-14T03:53:56.064 INFO:tasks.workunit.client.0.smithi110.stdout:665 2024-09-14T03:53:56.065 INFO:tasks.workunit.client.0.smithi110.stdout:666 2024-09-14T03:53:56.066 INFO:tasks.workunit.client.0.smithi110.stdout:667 2024-09-14T03:53:56.067 INFO:tasks.workunit.client.0.smithi110.stdout:668 2024-09-14T03:53:56.068 INFO:tasks.workunit.client.0.smithi110.stdout:669 2024-09-14T03:53:56.069 INFO:tasks.workunit.client.0.smithi110.stdout:670 2024-09-14T03:53:56.070 INFO:tasks.workunit.client.0.smithi110.stdout:671 2024-09-14T03:53:56.071 INFO:tasks.workunit.client.0.smithi110.stdout:672 2024-09-14T03:53:56.072 INFO:tasks.workunit.client.0.smithi110.stdout:673 2024-09-14T03:53:56.073 INFO:tasks.workunit.client.0.smithi110.stdout:674 2024-09-14T03:53:56.074 INFO:tasks.workunit.client.0.smithi110.stdout:675 2024-09-14T03:53:56.075 INFO:tasks.workunit.client.0.smithi110.stdout:676 2024-09-14T03:53:56.076 INFO:tasks.workunit.client.0.smithi110.stdout:677 2024-09-14T03:53:56.077 INFO:tasks.workunit.client.0.smithi110.stdout:678 2024-09-14T03:53:56.077 INFO:tasks.workunit.client.0.smithi110.stdout:679 2024-09-14T03:53:56.078 INFO:tasks.workunit.client.0.smithi110.stdout:680 2024-09-14T03:53:56.079 INFO:tasks.workunit.client.0.smithi110.stdout:681 2024-09-14T03:53:56.080 INFO:tasks.workunit.client.0.smithi110.stdout:682 2024-09-14T03:53:56.081 INFO:tasks.workunit.client.0.smithi110.stdout:683 2024-09-14T03:53:56.082 INFO:tasks.workunit.client.0.smithi110.stdout:684 2024-09-14T03:53:56.083 INFO:tasks.workunit.client.0.smithi110.stdout:685 2024-09-14T03:53:56.084 INFO:tasks.workunit.client.0.smithi110.stdout:686 2024-09-14T03:53:56.085 INFO:tasks.workunit.client.0.smithi110.stdout:687 2024-09-14T03:53:56.086 INFO:tasks.workunit.client.0.smithi110.stdout:688 2024-09-14T03:53:56.087 INFO:tasks.workunit.client.0.smithi110.stdout:689 2024-09-14T03:53:56.088 INFO:tasks.workunit.client.0.smithi110.stdout:690 2024-09-14T03:53:56.089 INFO:tasks.workunit.client.0.smithi110.stdout:691 2024-09-14T03:53:56.090 INFO:tasks.workunit.client.0.smithi110.stdout:692 2024-09-14T03:53:56.091 INFO:tasks.workunit.client.0.smithi110.stdout:693 2024-09-14T03:53:56.092 INFO:tasks.workunit.client.0.smithi110.stdout:694 2024-09-14T03:53:56.093 INFO:tasks.workunit.client.0.smithi110.stdout:695 2024-09-14T03:53:56.098 INFO:tasks.workunit.client.0.smithi110.stdout:696 2024-09-14T03:53:56.098 INFO:tasks.workunit.client.0.smithi110.stdout:697 2024-09-14T03:53:56.098 INFO:tasks.workunit.client.0.smithi110.stdout:698 2024-09-14T03:53:56.098 INFO:tasks.workunit.client.0.smithi110.stdout:699 2024-09-14T03:53:56.099 INFO:tasks.workunit.client.0.smithi110.stdout:700 2024-09-14T03:53:56.099 INFO:tasks.workunit.client.0.smithi110.stdout:701 2024-09-14T03:53:56.099 INFO:tasks.workunit.client.0.smithi110.stdout:702 2024-09-14T03:53:56.100 INFO:tasks.workunit.client.0.smithi110.stdout:703 2024-09-14T03:53:56.101 INFO:tasks.workunit.client.0.smithi110.stdout:704 2024-09-14T03:53:56.102 INFO:tasks.workunit.client.0.smithi110.stdout:705 2024-09-14T03:53:56.103 INFO:tasks.workunit.client.0.smithi110.stdout:706 2024-09-14T03:53:56.104 INFO:tasks.workunit.client.0.smithi110.stdout:707 2024-09-14T03:53:56.105 INFO:tasks.workunit.client.0.smithi110.stdout:708 2024-09-14T03:53:56.106 INFO:tasks.workunit.client.0.smithi110.stdout:709 2024-09-14T03:53:56.107 INFO:tasks.workunit.client.0.smithi110.stdout:710 2024-09-14T03:53:56.108 INFO:tasks.workunit.client.0.smithi110.stdout:711 2024-09-14T03:53:56.109 INFO:tasks.workunit.client.0.smithi110.stdout:712 2024-09-14T03:53:56.110 INFO:tasks.workunit.client.0.smithi110.stdout:713 2024-09-14T03:53:56.111 INFO:tasks.workunit.client.0.smithi110.stdout:714 2024-09-14T03:53:56.112 INFO:tasks.workunit.client.0.smithi110.stdout:715 2024-09-14T03:53:56.113 INFO:tasks.workunit.client.0.smithi110.stdout:716 2024-09-14T03:53:56.113 INFO:tasks.workunit.client.0.smithi110.stdout:717 2024-09-14T03:53:56.114 INFO:tasks.workunit.client.0.smithi110.stdout:718 2024-09-14T03:53:56.115 INFO:tasks.workunit.client.0.smithi110.stdout:719 2024-09-14T03:53:56.117 INFO:tasks.workunit.client.0.smithi110.stdout:720 2024-09-14T03:53:56.117 INFO:tasks.workunit.client.0.smithi110.stdout:721 2024-09-14T03:53:56.118 INFO:tasks.workunit.client.0.smithi110.stdout:722 2024-09-14T03:53:56.119 INFO:tasks.workunit.client.0.smithi110.stdout:723 2024-09-14T03:53:56.120 INFO:tasks.workunit.client.0.smithi110.stdout:724 2024-09-14T03:53:56.121 INFO:tasks.workunit.client.0.smithi110.stdout:725 2024-09-14T03:53:56.122 INFO:tasks.workunit.client.0.smithi110.stdout:726 2024-09-14T03:53:56.123 INFO:tasks.workunit.client.0.smithi110.stdout:727 2024-09-14T03:53:56.124 INFO:tasks.workunit.client.0.smithi110.stdout:728 2024-09-14T03:53:56.125 INFO:tasks.workunit.client.0.smithi110.stdout:729 2024-09-14T03:53:56.126 INFO:tasks.workunit.client.0.smithi110.stdout:730 2024-09-14T03:53:56.127 INFO:tasks.workunit.client.0.smithi110.stdout:731 2024-09-14T03:53:56.128 INFO:tasks.workunit.client.0.smithi110.stdout:732 2024-09-14T03:53:56.129 INFO:tasks.workunit.client.0.smithi110.stdout:733 2024-09-14T03:53:56.130 INFO:tasks.workunit.client.0.smithi110.stdout:734 2024-09-14T03:53:56.131 INFO:tasks.workunit.client.0.smithi110.stdout:735 2024-09-14T03:53:56.132 INFO:tasks.workunit.client.0.smithi110.stdout:736 2024-09-14T03:53:56.133 INFO:tasks.workunit.client.0.smithi110.stdout:737 2024-09-14T03:53:56.134 INFO:tasks.workunit.client.0.smithi110.stdout:738 2024-09-14T03:53:56.135 INFO:tasks.workunit.client.0.smithi110.stdout:739 2024-09-14T03:53:56.136 INFO:tasks.workunit.client.0.smithi110.stdout:740 2024-09-14T03:53:56.137 INFO:tasks.workunit.client.0.smithi110.stdout:741 2024-09-14T03:53:56.138 INFO:tasks.workunit.client.0.smithi110.stdout:742 2024-09-14T03:53:56.139 INFO:tasks.workunit.client.0.smithi110.stdout:743 2024-09-14T03:53:56.140 INFO:tasks.workunit.client.0.smithi110.stdout:744 2024-09-14T03:53:56.141 INFO:tasks.workunit.client.0.smithi110.stdout:745 2024-09-14T03:53:56.142 INFO:tasks.workunit.client.0.smithi110.stdout:746 2024-09-14T03:53:56.143 INFO:tasks.workunit.client.0.smithi110.stdout:747 2024-09-14T03:53:56.143 INFO:tasks.workunit.client.0.smithi110.stdout:748 2024-09-14T03:53:56.144 INFO:tasks.workunit.client.0.smithi110.stdout:749 2024-09-14T03:53:56.145 INFO:tasks.workunit.client.0.smithi110.stdout:750 2024-09-14T03:53:56.146 INFO:tasks.workunit.client.0.smithi110.stdout:751 2024-09-14T03:53:56.147 INFO:tasks.workunit.client.0.smithi110.stdout:752 2024-09-14T03:53:56.148 INFO:tasks.workunit.client.0.smithi110.stdout:753 2024-09-14T03:53:56.149 INFO:tasks.workunit.client.0.smithi110.stdout:754 2024-09-14T03:53:56.150 INFO:tasks.workunit.client.0.smithi110.stdout:755 2024-09-14T03:53:56.151 INFO:tasks.workunit.client.0.smithi110.stdout:756 2024-09-14T03:53:56.152 INFO:tasks.workunit.client.0.smithi110.stdout:757 2024-09-14T03:53:56.153 INFO:tasks.workunit.client.0.smithi110.stdout:758 2024-09-14T03:53:56.154 INFO:tasks.workunit.client.0.smithi110.stdout:759 2024-09-14T03:53:56.155 INFO:tasks.workunit.client.0.smithi110.stdout:760 2024-09-14T03:53:56.156 INFO:tasks.workunit.client.0.smithi110.stdout:761 2024-09-14T03:53:56.157 INFO:tasks.workunit.client.0.smithi110.stdout:762 2024-09-14T03:53:56.158 INFO:tasks.workunit.client.0.smithi110.stdout:763 2024-09-14T03:53:56.159 INFO:tasks.workunit.client.0.smithi110.stdout:764 2024-09-14T03:53:56.160 INFO:tasks.workunit.client.0.smithi110.stdout:765 2024-09-14T03:53:56.161 INFO:tasks.workunit.client.0.smithi110.stdout:766 2024-09-14T03:53:56.162 INFO:tasks.workunit.client.0.smithi110.stdout:767 2024-09-14T03:53:56.163 INFO:tasks.workunit.client.0.smithi110.stdout:768 2024-09-14T03:53:56.164 INFO:tasks.workunit.client.0.smithi110.stdout:769 2024-09-14T03:53:56.165 INFO:tasks.workunit.client.0.smithi110.stdout:770 2024-09-14T03:53:56.166 INFO:tasks.workunit.client.0.smithi110.stdout:771 2024-09-14T03:53:56.167 INFO:tasks.workunit.client.0.smithi110.stdout:772 2024-09-14T03:53:56.168 INFO:tasks.workunit.client.0.smithi110.stdout:773 2024-09-14T03:53:56.169 INFO:tasks.workunit.client.0.smithi110.stdout:774 2024-09-14T03:53:56.170 INFO:tasks.workunit.client.0.smithi110.stdout:775 2024-09-14T03:53:56.171 INFO:tasks.workunit.client.0.smithi110.stdout:776 2024-09-14T03:53:56.173 INFO:tasks.workunit.client.0.smithi110.stdout:777 2024-09-14T03:53:56.173 INFO:tasks.workunit.client.0.smithi110.stdout:778 2024-09-14T03:53:56.175 INFO:tasks.workunit.client.0.smithi110.stdout:779 2024-09-14T03:53:56.175 INFO:tasks.workunit.client.0.smithi110.stdout:780 2024-09-14T03:53:56.176 INFO:tasks.workunit.client.0.smithi110.stdout:781 2024-09-14T03:53:56.177 INFO:tasks.workunit.client.0.smithi110.stdout:782 2024-09-14T03:53:56.179 INFO:tasks.workunit.client.0.smithi110.stdout:783 2024-09-14T03:53:56.179 INFO:tasks.workunit.client.0.smithi110.stdout:784 2024-09-14T03:53:56.181 INFO:tasks.workunit.client.0.smithi110.stdout:785 2024-09-14T03:53:56.181 INFO:tasks.workunit.client.0.smithi110.stdout:786 2024-09-14T03:53:56.183 INFO:tasks.workunit.client.0.smithi110.stdout:787 2024-09-14T03:53:56.184 INFO:tasks.workunit.client.0.smithi110.stdout:788 2024-09-14T03:53:56.185 INFO:tasks.workunit.client.0.smithi110.stdout:789 2024-09-14T03:53:56.186 INFO:tasks.workunit.client.0.smithi110.stdout:790 2024-09-14T03:53:56.186 INFO:tasks.workunit.client.0.smithi110.stdout:791 2024-09-14T03:53:56.187 INFO:tasks.workunit.client.0.smithi110.stdout:792 2024-09-14T03:53:56.188 INFO:tasks.workunit.client.0.smithi110.stdout:793 2024-09-14T03:53:56.189 INFO:tasks.workunit.client.0.smithi110.stdout:794 2024-09-14T03:53:56.190 INFO:tasks.workunit.client.0.smithi110.stdout:795 2024-09-14T03:53:56.191 INFO:tasks.workunit.client.0.smithi110.stdout:796 2024-09-14T03:53:56.192 INFO:tasks.workunit.client.0.smithi110.stdout:797 2024-09-14T03:53:56.193 INFO:tasks.workunit.client.0.smithi110.stdout:798 2024-09-14T03:53:56.194 INFO:tasks.workunit.client.0.smithi110.stdout:799 2024-09-14T03:53:56.195 INFO:tasks.workunit.client.0.smithi110.stdout:800 2024-09-14T03:53:56.196 INFO:tasks.workunit.client.0.smithi110.stdout:801 2024-09-14T03:53:56.197 INFO:tasks.workunit.client.0.smithi110.stdout:802 2024-09-14T03:53:56.198 INFO:tasks.workunit.client.0.smithi110.stdout:803 2024-09-14T03:53:56.199 INFO:tasks.workunit.client.0.smithi110.stdout:804 2024-09-14T03:53:56.200 INFO:tasks.workunit.client.0.smithi110.stdout:805 2024-09-14T03:53:56.201 INFO:tasks.workunit.client.0.smithi110.stdout:806 2024-09-14T03:53:56.202 INFO:tasks.workunit.client.0.smithi110.stdout:807 2024-09-14T03:53:56.203 INFO:tasks.workunit.client.0.smithi110.stdout:808 2024-09-14T03:53:56.204 INFO:tasks.workunit.client.0.smithi110.stdout:809 2024-09-14T03:53:56.206 INFO:tasks.workunit.client.0.smithi110.stdout:810 2024-09-14T03:53:56.206 INFO:tasks.workunit.client.0.smithi110.stdout:811 2024-09-14T03:53:56.208 INFO:tasks.workunit.client.0.smithi110.stdout:812 2024-09-14T03:53:56.209 INFO:tasks.workunit.client.0.smithi110.stdout:813 2024-09-14T03:53:56.209 INFO:tasks.workunit.client.0.smithi110.stdout:814 2024-09-14T03:53:56.211 INFO:tasks.workunit.client.0.smithi110.stdout:815 2024-09-14T03:53:56.211 INFO:tasks.workunit.client.0.smithi110.stdout:816 2024-09-14T03:53:56.212 INFO:tasks.workunit.client.0.smithi110.stdout:817 2024-09-14T03:53:56.213 INFO:tasks.workunit.client.0.smithi110.stdout:818 2024-09-14T03:53:56.214 INFO:tasks.workunit.client.0.smithi110.stdout:819 2024-09-14T03:53:56.215 INFO:tasks.workunit.client.0.smithi110.stdout:820 2024-09-14T03:53:56.216 INFO:tasks.workunit.client.0.smithi110.stdout:821 2024-09-14T03:53:56.217 INFO:tasks.workunit.client.0.smithi110.stdout:822 2024-09-14T03:53:56.219 INFO:tasks.workunit.client.0.smithi110.stdout:823 2024-09-14T03:53:56.220 INFO:tasks.workunit.client.0.smithi110.stdout:824 2024-09-14T03:53:56.221 INFO:tasks.workunit.client.0.smithi110.stdout:825 2024-09-14T03:53:56.222 INFO:tasks.workunit.client.0.smithi110.stdout:826 2024-09-14T03:53:56.222 INFO:tasks.workunit.client.0.smithi110.stdout:827 2024-09-14T03:53:56.223 INFO:tasks.workunit.client.0.smithi110.stdout:828 2024-09-14T03:53:56.224 INFO:tasks.workunit.client.0.smithi110.stdout:829 2024-09-14T03:53:56.225 INFO:tasks.workunit.client.0.smithi110.stdout:830 2024-09-14T03:53:56.226 INFO:tasks.workunit.client.0.smithi110.stdout:831 2024-09-14T03:53:56.228 INFO:tasks.workunit.client.0.smithi110.stdout:832 2024-09-14T03:53:56.229 INFO:tasks.workunit.client.0.smithi110.stdout:833 2024-09-14T03:53:56.230 INFO:tasks.workunit.client.0.smithi110.stdout:834 2024-09-14T03:53:56.231 INFO:tasks.workunit.client.0.smithi110.stdout:835 2024-09-14T03:53:56.232 INFO:tasks.workunit.client.0.smithi110.stdout:836 2024-09-14T03:53:56.233 INFO:tasks.workunit.client.0.smithi110.stdout:837 2024-09-14T03:53:56.234 INFO:tasks.workunit.client.0.smithi110.stdout:838 2024-09-14T03:53:56.235 INFO:tasks.workunit.client.0.smithi110.stdout:839 2024-09-14T03:53:56.235 INFO:tasks.workunit.client.0.smithi110.stdout:840 2024-09-14T03:53:56.236 INFO:tasks.workunit.client.0.smithi110.stdout:841 2024-09-14T03:53:56.237 INFO:tasks.workunit.client.0.smithi110.stdout:842 2024-09-14T03:53:56.238 INFO:tasks.workunit.client.0.smithi110.stdout:843 2024-09-14T03:53:56.239 INFO:tasks.workunit.client.0.smithi110.stdout:844 2024-09-14T03:53:56.240 INFO:tasks.workunit.client.0.smithi110.stdout:845 2024-09-14T03:53:56.241 INFO:tasks.workunit.client.0.smithi110.stdout:846 2024-09-14T03:53:56.242 INFO:tasks.workunit.client.0.smithi110.stdout:847 2024-09-14T03:53:56.244 INFO:tasks.workunit.client.0.smithi110.stdout:848 2024-09-14T03:53:56.245 INFO:tasks.workunit.client.0.smithi110.stdout:849 2024-09-14T03:53:56.245 INFO:tasks.workunit.client.0.smithi110.stdout:850 2024-09-14T03:53:56.246 INFO:tasks.workunit.client.0.smithi110.stdout:851 2024-09-14T03:53:56.248 INFO:tasks.workunit.client.0.smithi110.stdout:852 2024-09-14T03:53:56.249 INFO:tasks.workunit.client.0.smithi110.stdout:853 2024-09-14T03:53:56.250 INFO:tasks.workunit.client.0.smithi110.stdout:854 2024-09-14T03:53:56.251 INFO:tasks.workunit.client.0.smithi110.stdout:855 2024-09-14T03:53:56.252 INFO:tasks.workunit.client.0.smithi110.stdout:856 2024-09-14T03:53:56.253 INFO:tasks.workunit.client.0.smithi110.stdout:857 2024-09-14T03:53:56.254 INFO:tasks.workunit.client.0.smithi110.stdout:858 2024-09-14T03:53:56.255 INFO:tasks.workunit.client.0.smithi110.stdout:859 2024-09-14T03:53:56.256 INFO:tasks.workunit.client.0.smithi110.stdout:860 2024-09-14T03:53:56.257 INFO:tasks.workunit.client.0.smithi110.stdout:861 2024-09-14T03:53:56.258 INFO:tasks.workunit.client.0.smithi110.stdout:862 2024-09-14T03:53:56.259 INFO:tasks.workunit.client.0.smithi110.stdout:863 2024-09-14T03:53:56.260 INFO:tasks.workunit.client.0.smithi110.stdout:864 2024-09-14T03:53:56.261 INFO:tasks.workunit.client.0.smithi110.stdout:865 2024-09-14T03:53:56.262 INFO:tasks.workunit.client.0.smithi110.stdout:866 2024-09-14T03:53:56.263 INFO:tasks.workunit.client.0.smithi110.stdout:867 2024-09-14T03:53:56.264 INFO:tasks.workunit.client.0.smithi110.stdout:868 2024-09-14T03:53:56.265 INFO:tasks.workunit.client.0.smithi110.stdout:869 2024-09-14T03:53:56.266 INFO:tasks.workunit.client.0.smithi110.stdout:870 2024-09-14T03:53:56.267 INFO:tasks.workunit.client.0.smithi110.stdout:871 2024-09-14T03:53:56.268 INFO:tasks.workunit.client.0.smithi110.stdout:872 2024-09-14T03:53:56.269 INFO:tasks.workunit.client.0.smithi110.stdout:873 2024-09-14T03:53:56.270 INFO:tasks.workunit.client.0.smithi110.stdout:874 2024-09-14T03:53:56.271 INFO:tasks.workunit.client.0.smithi110.stdout:875 2024-09-14T03:53:56.272 INFO:tasks.workunit.client.0.smithi110.stdout:876 2024-09-14T03:53:56.273 INFO:tasks.workunit.client.0.smithi110.stdout:877 2024-09-14T03:53:56.274 INFO:tasks.workunit.client.0.smithi110.stdout:878 2024-09-14T03:53:56.275 INFO:tasks.workunit.client.0.smithi110.stdout:879 2024-09-14T03:53:56.276 INFO:tasks.workunit.client.0.smithi110.stdout:880 2024-09-14T03:53:56.277 INFO:tasks.workunit.client.0.smithi110.stdout:881 2024-09-14T03:53:56.278 INFO:tasks.workunit.client.0.smithi110.stdout:882 2024-09-14T03:53:56.279 INFO:tasks.workunit.client.0.smithi110.stdout:883 2024-09-14T03:53:56.280 INFO:tasks.workunit.client.0.smithi110.stdout:884 2024-09-14T03:53:56.281 INFO:tasks.workunit.client.0.smithi110.stdout:885 2024-09-14T03:53:56.282 INFO:tasks.workunit.client.0.smithi110.stdout:886 2024-09-14T03:53:56.283 INFO:tasks.workunit.client.0.smithi110.stdout:887 2024-09-14T03:53:56.285 INFO:tasks.workunit.client.0.smithi110.stdout:888 2024-09-14T03:53:56.286 INFO:tasks.workunit.client.0.smithi110.stdout:889 2024-09-14T03:53:56.288 INFO:tasks.workunit.client.0.smithi110.stdout:890 2024-09-14T03:53:56.290 INFO:tasks.workunit.client.0.smithi110.stdout:891 2024-09-14T03:53:56.292 INFO:tasks.workunit.client.0.smithi110.stdout:892 2024-09-14T03:53:56.293 INFO:tasks.workunit.client.0.smithi110.stdout:893 2024-09-14T03:53:56.294 INFO:tasks.workunit.client.0.smithi110.stdout:894 2024-09-14T03:53:56.295 INFO:tasks.workunit.client.0.smithi110.stdout:895 2024-09-14T03:53:56.296 INFO:tasks.workunit.client.0.smithi110.stdout:896 2024-09-14T03:53:56.297 INFO:tasks.workunit.client.0.smithi110.stdout:897 2024-09-14T03:53:56.298 INFO:tasks.workunit.client.0.smithi110.stdout:898 2024-09-14T03:53:56.299 INFO:tasks.workunit.client.0.smithi110.stdout:899 2024-09-14T03:53:56.300 INFO:tasks.workunit.client.0.smithi110.stdout:900 2024-09-14T03:53:56.301 INFO:tasks.workunit.client.0.smithi110.stdout:901 2024-09-14T03:53:56.302 INFO:tasks.workunit.client.0.smithi110.stdout:902 2024-09-14T03:53:56.303 INFO:tasks.workunit.client.0.smithi110.stdout:903 2024-09-14T03:53:56.304 INFO:tasks.workunit.client.0.smithi110.stdout:904 2024-09-14T03:53:56.305 INFO:tasks.workunit.client.0.smithi110.stdout:905 2024-09-14T03:53:56.306 INFO:tasks.workunit.client.0.smithi110.stdout:906 2024-09-14T03:53:56.307 INFO:tasks.workunit.client.0.smithi110.stdout:907 2024-09-14T03:53:56.308 INFO:tasks.workunit.client.0.smithi110.stdout:908 2024-09-14T03:53:56.309 INFO:tasks.workunit.client.0.smithi110.stdout:909 2024-09-14T03:53:56.311 INFO:tasks.workunit.client.0.smithi110.stdout:910 2024-09-14T03:53:56.312 INFO:tasks.workunit.client.0.smithi110.stdout:911 2024-09-14T03:53:56.313 INFO:tasks.workunit.client.0.smithi110.stdout:912 2024-09-14T03:53:56.314 INFO:tasks.workunit.client.0.smithi110.stdout:913 2024-09-14T03:53:56.315 INFO:tasks.workunit.client.0.smithi110.stdout:914 2024-09-14T03:53:56.316 INFO:tasks.workunit.client.0.smithi110.stdout:915 2024-09-14T03:53:56.317 INFO:tasks.workunit.client.0.smithi110.stdout:916 2024-09-14T03:53:56.318 INFO:tasks.workunit.client.0.smithi110.stdout:917 2024-09-14T03:53:56.319 INFO:tasks.workunit.client.0.smithi110.stdout:918 2024-09-14T03:53:56.320 INFO:tasks.workunit.client.0.smithi110.stdout:919 2024-09-14T03:53:56.321 INFO:tasks.workunit.client.0.smithi110.stdout:920 2024-09-14T03:53:56.322 INFO:tasks.workunit.client.0.smithi110.stdout:921 2024-09-14T03:53:56.323 INFO:tasks.workunit.client.0.smithi110.stdout:922 2024-09-14T03:53:56.324 INFO:tasks.workunit.client.0.smithi110.stdout:923 2024-09-14T03:53:56.325 INFO:tasks.workunit.client.0.smithi110.stdout:924 2024-09-14T03:53:56.326 INFO:tasks.workunit.client.0.smithi110.stdout:925 2024-09-14T03:53:56.327 INFO:tasks.workunit.client.0.smithi110.stdout:926 2024-09-14T03:53:56.328 INFO:tasks.workunit.client.0.smithi110.stdout:927 2024-09-14T03:53:56.329 INFO:tasks.workunit.client.0.smithi110.stdout:928 2024-09-14T03:53:56.331 INFO:tasks.workunit.client.0.smithi110.stdout:929 2024-09-14T03:53:56.332 INFO:tasks.workunit.client.0.smithi110.stdout:930 2024-09-14T03:53:56.333 INFO:tasks.workunit.client.0.smithi110.stdout:931 2024-09-14T03:53:56.334 INFO:tasks.workunit.client.0.smithi110.stdout:932 2024-09-14T03:53:56.335 INFO:tasks.workunit.client.0.smithi110.stdout:933 2024-09-14T03:53:56.336 INFO:tasks.workunit.client.0.smithi110.stdout:934 2024-09-14T03:53:56.337 INFO:tasks.workunit.client.0.smithi110.stdout:935 2024-09-14T03:53:56.338 INFO:tasks.workunit.client.0.smithi110.stdout:936 2024-09-14T03:53:56.339 INFO:tasks.workunit.client.0.smithi110.stdout:937 2024-09-14T03:53:56.340 INFO:tasks.workunit.client.0.smithi110.stdout:938 2024-09-14T03:53:56.341 INFO:tasks.workunit.client.0.smithi110.stdout:939 2024-09-14T03:53:56.342 INFO:tasks.workunit.client.0.smithi110.stdout:940 2024-09-14T03:53:56.343 INFO:tasks.workunit.client.0.smithi110.stdout:941 2024-09-14T03:53:56.344 INFO:tasks.workunit.client.0.smithi110.stdout:942 2024-09-14T03:53:56.345 INFO:tasks.workunit.client.0.smithi110.stdout:943 2024-09-14T03:53:56.346 INFO:tasks.workunit.client.0.smithi110.stdout:944 2024-09-14T03:53:56.347 INFO:tasks.workunit.client.0.smithi110.stdout:945 2024-09-14T03:53:56.348 INFO:tasks.workunit.client.0.smithi110.stdout:946 2024-09-14T03:53:56.349 INFO:tasks.workunit.client.0.smithi110.stdout:947 2024-09-14T03:53:56.350 INFO:tasks.workunit.client.0.smithi110.stdout:948 2024-09-14T03:53:56.351 INFO:tasks.workunit.client.0.smithi110.stdout:949 2024-09-14T03:53:56.352 INFO:tasks.workunit.client.0.smithi110.stdout:950 2024-09-14T03:53:56.353 INFO:tasks.workunit.client.0.smithi110.stdout:951 2024-09-14T03:53:56.354 INFO:tasks.workunit.client.0.smithi110.stdout:952 2024-09-14T03:53:56.355 INFO:tasks.workunit.client.0.smithi110.stdout:953 2024-09-14T03:53:56.356 INFO:tasks.workunit.client.0.smithi110.stdout:954 2024-09-14T03:53:56.357 INFO:tasks.workunit.client.0.smithi110.stdout:955 2024-09-14T03:53:56.358 INFO:tasks.workunit.client.0.smithi110.stdout:956 2024-09-14T03:53:56.359 INFO:tasks.workunit.client.0.smithi110.stdout:957 2024-09-14T03:53:56.360 INFO:tasks.workunit.client.0.smithi110.stdout:958 2024-09-14T03:53:56.361 INFO:tasks.workunit.client.0.smithi110.stdout:959 2024-09-14T03:53:56.362 INFO:tasks.workunit.client.0.smithi110.stdout:960 2024-09-14T03:53:56.363 INFO:tasks.workunit.client.0.smithi110.stdout:961 2024-09-14T03:53:56.364 INFO:tasks.workunit.client.0.smithi110.stdout:962 2024-09-14T03:53:56.365 INFO:tasks.workunit.client.0.smithi110.stdout:963 2024-09-14T03:53:56.367 INFO:tasks.workunit.client.0.smithi110.stdout:964 2024-09-14T03:53:56.368 INFO:tasks.workunit.client.0.smithi110.stdout:965 2024-09-14T03:53:56.369 INFO:tasks.workunit.client.0.smithi110.stdout:966 2024-09-14T03:53:56.370 INFO:tasks.workunit.client.0.smithi110.stdout:967 2024-09-14T03:53:56.371 INFO:tasks.workunit.client.0.smithi110.stdout:968 2024-09-14T03:53:56.372 INFO:tasks.workunit.client.0.smithi110.stdout:969 2024-09-14T03:53:56.373 INFO:tasks.workunit.client.0.smithi110.stdout:970 2024-09-14T03:53:56.374 INFO:tasks.workunit.client.0.smithi110.stdout:971 2024-09-14T03:53:56.375 INFO:tasks.workunit.client.0.smithi110.stdout:972 2024-09-14T03:53:56.376 INFO:tasks.workunit.client.0.smithi110.stdout:973 2024-09-14T03:53:56.377 INFO:tasks.workunit.client.0.smithi110.stdout:974 2024-09-14T03:53:56.378 INFO:tasks.workunit.client.0.smithi110.stdout:975 2024-09-14T03:53:56.379 INFO:tasks.workunit.client.0.smithi110.stdout:976 2024-09-14T03:53:56.380 INFO:tasks.workunit.client.0.smithi110.stdout:977 2024-09-14T03:53:56.381 INFO:tasks.workunit.client.0.smithi110.stdout:978 2024-09-14T03:53:56.382 INFO:tasks.workunit.client.0.smithi110.stdout:979 2024-09-14T03:53:56.383 INFO:tasks.workunit.client.0.smithi110.stdout:980 2024-09-14T03:53:56.384 INFO:tasks.workunit.client.0.smithi110.stdout:981 2024-09-14T03:53:56.385 INFO:tasks.workunit.client.0.smithi110.stdout:982 2024-09-14T03:53:56.386 INFO:tasks.workunit.client.0.smithi110.stdout:983 2024-09-14T03:53:56.387 INFO:tasks.workunit.client.0.smithi110.stdout:984 2024-09-14T03:53:56.388 INFO:tasks.workunit.client.0.smithi110.stdout:985 2024-09-14T03:53:56.389 INFO:tasks.workunit.client.0.smithi110.stdout:986 2024-09-14T03:53:56.390 INFO:tasks.workunit.client.0.smithi110.stdout:987 2024-09-14T03:53:56.391 INFO:tasks.workunit.client.0.smithi110.stdout:988 2024-09-14T03:53:56.392 INFO:tasks.workunit.client.0.smithi110.stdout:989 2024-09-14T03:53:56.393 INFO:tasks.workunit.client.0.smithi110.stdout:990 2024-09-14T03:53:56.394 INFO:tasks.workunit.client.0.smithi110.stdout:991 2024-09-14T03:53:56.395 INFO:tasks.workunit.client.0.smithi110.stdout:992 2024-09-14T03:53:56.396 INFO:tasks.workunit.client.0.smithi110.stdout:993 2024-09-14T03:53:56.397 INFO:tasks.workunit.client.0.smithi110.stdout:994 2024-09-14T03:53:56.398 INFO:tasks.workunit.client.0.smithi110.stdout:995 2024-09-14T03:53:56.399 INFO:tasks.workunit.client.0.smithi110.stdout:996 2024-09-14T03:53:56.400 INFO:tasks.workunit.client.0.smithi110.stdout:997 2024-09-14T03:53:56.401 INFO:tasks.workunit.client.0.smithi110.stdout:998 2024-09-14T03:53:56.402 INFO:tasks.workunit.client.0.smithi110.stdout:999 2024-09-14T03:53:56.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:56 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:56.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:56 smithi110 ceph-mon[86423]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T03:53:56.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:56 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1621203528' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-14T03:53:56.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:56 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:56.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:56 smithi110 ceph-mon[88262]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T03:53:56.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:56 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1621203528' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-14T03:53:56.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:56 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:53:56.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:56 smithi195 ceph-mon[82388]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T03:53:56.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:56 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1621203528' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-14T03:53:57.294 INFO:tasks.workunit.client.0.smithi110.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-14T03:53:57.294 INFO:tasks.workunit.client.0.smithi110.stdout:00000010 69 73 |is| 2024-09-14T03:53:57.294 INFO:tasks.workunit.client.0.smithi110.stdout:00000012 2024-09-14T03:53:57.294 INFO:tasks.workunit.client.0.smithi110.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-14T03:53:57.294 INFO:tasks.workunit.client.0.smithi110.stdout:00000010 69 73 |is| 2024-09-14T03:53:57.294 INFO:tasks.workunit.client.0.smithi110.stdout:00000012 2024-09-14T03:53:57.294 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsHello.WriteReturnData (4020 ms) 2024-09-14T03:53:57.294 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsHello.Loud 2024-09-14T03:53:57.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:57 smithi110 ceph-mon[88262]: pgmap v1123: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:57.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:57 smithi110 ceph-mon[88262]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T03:53:57.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:57 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:57.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:57.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:57 smithi110 ceph-mon[86423]: pgmap v1123: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:57.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:57 smithi110 ceph-mon[86423]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T03:53:57.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:57 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:57.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:57.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:57 smithi195 ceph-mon[82388]: pgmap v1123: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:57.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:57 smithi195 ceph-mon[82388]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T03:53:57.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:57 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:53:57.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:58.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:58 smithi110 ceph-mon[88262]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T03:53:58.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:58 smithi110 ceph-mon[86423]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T03:53:58.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:58 smithi195 ceph-mon[82388]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T03:53:59.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:59 smithi110 ceph-mon[88262]: pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:59.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:59 smithi110 ceph-mon[88262]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T03:53:59.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:59 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2036784083' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:59.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:53:59 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:59.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:59 smithi110 ceph-mon[86423]: pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:59.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:59 smithi110 ceph-mon[86423]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T03:53:59.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:59 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2036784083' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:59.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:53:59 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:59.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:59 smithi195 ceph-mon[82388]: pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:53:59.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:59 smithi195 ceph-mon[82388]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T03:53:59.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:59 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2036784083' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:53:59.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:53:59 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:00.325 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsHello.Loud (3031 ms) 2024-09-14T03:54:00.325 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsHello.BadMethods 2024-09-14T03:54:00.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:00 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:00.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:00 smithi110 ceph-mon[88262]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T03:54:00.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:00 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:00.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:00 smithi110 ceph-mon[86423]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T03:54:00.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:00 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:00.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:00 smithi195 ceph-mon[82388]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T03:54:00.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:54:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:54:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:01.597 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:54:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:54:01] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.51.0" 2024-09-14T03:54:01.598 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:01 smithi110 ceph-mon[86423]: pgmap v1129: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:01.598 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:01 smithi110 ceph-mon[86423]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T03:54:01.598 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:01 smithi110 ceph-mon[88262]: pgmap v1129: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:01.598 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:01 smithi110 ceph-mon[88262]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T03:54:01.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:01 smithi195 ceph-mon[82388]: pgmap v1129: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:01.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:01 smithi195 ceph-mon[82388]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T03:54:02.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:02 smithi110 ceph-mon[86423]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T03:54:02.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:02 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/751820668' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:02.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:02 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:02.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:02 smithi110 ceph-mon[88262]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T03:54:02.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:02 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/751820668' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:02.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:02 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:02.697 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:54:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[106966]: 2024-09-14T03:54:02.352+0000 7fdb8303c640 -1 osd.3 pg_epoch: 984 pg[38.2( v 984'1 (0'0,984'1] local-lis/les=983/984 n=1 ec=983/983 lis/c=983/983 les/c/f=984/984/0 sis=983) [3,7,5] r=0 lpr=983 crt=984'1 lcod 0'0 mlcod 0'0 active+clean] method hello.bad_reader tried to read object but is not marked RD 2024-09-14T03:54:02.697 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:54:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[106966]: 2024-09-14T03:54:02.353+0000 7fdb7f034640 -1 osd.3 pg_epoch: 984 pg[38.2( v 984'2 (0'0,984'2] local-lis/les=983/984 n=1 ec=983/983 lis/c=983/983 les/c/f=984/984/0 sis=983) [3,7,5] r=0 lpr=983 crt=984'1 lcod 984'1 mlcod 984'1 active+clean] method hello.bad_writer tried to update object but is not marked WR 2024-09-14T03:54:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:02 smithi195 ceph-mon[82388]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T03:54:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:02 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/751820668' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:02.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:02 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:03.337 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsHello.BadMethods (3011 ms) 2024-09-14T03:54:03.337 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsHello.Filter 2024-09-14T03:54:03.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[88262]: pgmap v1132: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:03.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:03.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:03.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[88262]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T03:54:03.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:03.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:03.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:54:03.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:54:03.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[86423]: pgmap v1132: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:03.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:03.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:03.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[86423]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T03:54:03.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:03.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:03.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:54:03.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:03 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:54:03.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:03 smithi195 ceph-mon[82388]: pgmap v1132: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:03.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:03 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:03.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:03 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:03.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:03 smithi195 ceph-mon[82388]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T03:54:03.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:03 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:03.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:03 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:03.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:03 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:54:03.710 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:03 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:54:04.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:04 smithi110 ceph-mon[88262]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T03:54:04.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:04 smithi110 ceph-mon[86423]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T03:54:04.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:04 smithi195 ceph-mon[82388]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T03:54:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[88262]: pgmap v1135: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:54:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[88262]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T03:54:05.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4044682237' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:05.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[86423]: pgmap v1135: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:54:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[86423]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T03:54:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4044682237' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:05.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:05 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:05.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:05 smithi195 ceph-mon[82388]: pgmap v1135: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:05 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:05 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:54:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:05 smithi195 ceph-mon[82388]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T03:54:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:05 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4044682237' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:05.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:05 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:06.362 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsHello.Filter (3026 ms) 2024-09-14T03:54:06.362 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 6 tests from ClsHello (18981 ms total) 2024-09-14T03:54:06.363 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:54:06.363 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:54:06.363 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 6 tests from 1 test suite ran. (18981 ms total) 2024-09-14T03:54:06.363 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 6 tests. 2024-09-14T03:54:06.365 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:54:06.365 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:54:06.435 INFO:tasks.workunit:Running workunit cls/test_cls_journal.sh... 2024-09-14T03:54:06.435 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_journal.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh 2024-09-14T03:54:06.504 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:54:06.505 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 27 tests from 1 test suite. 2024-09-14T03:54:06.505 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:54:06.505 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 27 tests from TestClsJournal 2024-09-14T03:54:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:06 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:06.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:06 smithi110 ceph-mon[88262]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T03:54:06.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:06 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:06.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:06 smithi110 ceph-mon[86423]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T03:54:06.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:06 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117597-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:06.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:06 smithi195 ceph-mon[82388]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T03:54:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:07 smithi110 ceph-mon[88262]: pgmap v1138: 137 pgs: 3 creating+peering, 27 unknown, 107 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:07.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:07 smithi110 ceph-mon[88262]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T03:54:07.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:07 smithi110 ceph-mon[86423]: pgmap v1138: 137 pgs: 3 creating+peering, 27 unknown, 107 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:07.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:07 smithi110 ceph-mon[86423]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T03:54:07.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:07 smithi195 ceph-mon[82388]: pgmap v1138: 137 pgs: 3 creating+peering, 27 unknown, 107 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:07.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:07 smithi195 ceph-mon[82388]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T03:54:08.371 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.Create 2024-09-14T03:54:08.386 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.Create (15 ms) 2024-09-14T03:54:08.386 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.MinimumSet 2024-09-14T03:54:08.392 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.MinimumSet (6 ms) 2024-09-14T03:54:08.393 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.MinimumSetStale 2024-09-14T03:54:08.398 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.MinimumSetStale (5 ms) 2024-09-14T03:54:08.398 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.MinimumSetOrderConstraint 2024-09-14T03:54:08.401 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.MinimumSetOrderConstraint (4 ms) 2024-09-14T03:54:08.401 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ActiveSet 2024-09-14T03:54:08.404 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ActiveSet (2 ms) 2024-09-14T03:54:08.404 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ActiveSetStale 2024-09-14T03:54:08.407 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ActiveSetStale (3 ms) 2024-09-14T03:54:08.407 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.CreateDuplicate 2024-09-14T03:54:08.409 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.CreateDuplicate (3 ms) 2024-09-14T03:54:08.409 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.GetClient 2024-09-14T03:54:08.412 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.GetClient (2 ms) 2024-09-14T03:54:08.412 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientRegister 2024-09-14T03:54:08.414 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientRegister (3 ms) 2024-09-14T03:54:08.414 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientRegisterDuplicate 2024-09-14T03:54:08.418 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientRegisterDuplicate (3 ms) 2024-09-14T03:54:08.418 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientUpdateData 2024-09-14T03:54:08.421 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientUpdateData (4 ms) 2024-09-14T03:54:08.422 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientUpdateState 2024-09-14T03:54:08.426 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientUpdateState (5 ms) 2024-09-14T03:54:08.426 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientUnregister 2024-09-14T03:54:08.429 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientUnregister (3 ms) 2024-09-14T03:54:08.429 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientUnregisterDNE 2024-09-14T03:54:08.432 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientUnregisterDNE (3 ms) 2024-09-14T03:54:08.432 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientUnregisterPruneTags 2024-09-14T03:54:08.527 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientUnregisterPruneTags (95 ms) 2024-09-14T03:54:08.527 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientCommit 2024-09-14T03:54:08.531 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientCommit (3 ms) 2024-09-14T03:54:08.531 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientCommitInvalid 2024-09-14T03:54:08.533 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientCommitInvalid (3 ms) 2024-09-14T03:54:08.533 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientCommitDNE 2024-09-14T03:54:08.534 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientCommitDNE (1 ms) 2024-09-14T03:54:08.534 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.ClientList 2024-09-14T03:54:08.589 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.ClientList (55 ms) 2024-09-14T03:54:08.589 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.GetNextTagTid 2024-09-14T03:54:08.593 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.GetNextTagTid (4 ms) 2024-09-14T03:54:08.593 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.TagCreate 2024-09-14T03:54:08.601 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.TagCreate (8 ms) 2024-09-14T03:54:08.601 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.TagCreatePrunesTags 2024-09-14T03:54:08.608 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.TagCreatePrunesTags (7 ms) 2024-09-14T03:54:08.608 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.TagList 2024-09-14T03:54:08.691 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.TagList (82 ms) 2024-09-14T03:54:08.691 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.GuardAppend 2024-09-14T03:54:08.692 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.GuardAppend (2 ms) 2024-09-14T03:54:08.693 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.GuardAppendDNE 2024-09-14T03:54:08.693 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.GuardAppendDNE (1 ms) 2024-09-14T03:54:08.694 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.GuardAppendOverflow 2024-09-14T03:54:08.695 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.GuardAppendOverflow (2 ms) 2024-09-14T03:54:08.695 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsJournal.Append 2024-09-14T03:54:08.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:08 smithi110 ceph-mon[88262]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T03:54:08.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:08 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2526016804' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:08.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:08 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:08.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:08 smithi110 ceph-mon[86423]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T03:54:08.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:08 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2526016804' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:08.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:08 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:08.701 INFO:tasks.workunit.client.0.smithi110.stdout:padding is not enabled 2024-09-14T03:54:08.701 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsJournal.Append (6 ms) 2024-09-14T03:54:08.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:08 smithi195 ceph-mon[82388]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T03:54:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:08 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2526016804' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:08.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:08 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:09.379 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 27 tests from TestClsJournal (330 ms total) 2024-09-14T03:54:09.379 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:54:09.379 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:54:09.379 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 27 tests from 1 test suite ran. (2874 ms total) 2024-09-14T03:54:09.379 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 27 tests. 2024-09-14T03:54:09.381 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:54:09.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:54:09.450 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2024-09-14T03:54:09.450 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2024-09-14T03:54:09.519 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:54:09.519 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 11 tests from 1 test suite. 2024-09-14T03:54:09.519 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:54:09.520 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 11 tests from ClsLock 2024-09-14T03:54:09.520 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestMultiLocking 2024-09-14T03:54:09.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:09 smithi110 ceph-mon[86423]: pgmap v1141: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:09.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:09 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:09.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:09 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:09.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:09 smithi110 ceph-mon[86423]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T03:54:09.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:09 smithi110 ceph-mon[88262]: pgmap v1141: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:09.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:09 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:09.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:09 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:09.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:09 smithi110 ceph-mon[88262]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T03:54:09.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:09 smithi195 ceph-mon[82388]: pgmap v1141: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:09 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:09 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117820-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:09.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:09 smithi195 ceph-mon[82388]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T03:54:10.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:10 smithi110 ceph-mon[88262]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T03:54:10.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:10 smithi110 ceph-mon[86423]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T03:54:10.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:10 smithi195 ceph-mon[82388]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T03:54:10.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:54:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:54:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:11.418 INFO:tasks.workunit.client.0.smithi110.stdout:lock: mylock 2024-09-14T03:54:11.418 INFO:tasks.workunit.client.0.smithi110.stdout: lock_type: exclusive 2024-09-14T03:54:11.418 INFO:tasks.workunit.client.0.smithi110.stdout: tag: 2024-09-14T03:54:11.418 INFO:tasks.workunit.client.0.smithi110.stdout: lockers: 2024-09-14T03:54:11.418 INFO:tasks.workunit.client.0.smithi110.stdout: client.44735 expiration=2024-09-14T03:56:11.414665+0000 addr=v1:172.21.15.110:0/2121143186 cookie= 2024-09-14T03:54:11.422 INFO:tasks.workunit.client.0.smithi110.stdout:lock: mylock 2024-09-14T03:54:11.422 INFO:tasks.workunit.client.0.smithi110.stdout: lock_type: shared 2024-09-14T03:54:11.422 INFO:tasks.workunit.client.0.smithi110.stdout: tag: 2024-09-14T03:54:11.422 INFO:tasks.workunit.client.0.smithi110.stdout: lockers: 2024-09-14T03:54:11.422 INFO:tasks.workunit.client.0.smithi110.stdout: client.44735 expiration=2024-09-14T03:56:11.420594+0000 addr=v1:172.21.15.110:0/2121143186 cookie= 2024-09-14T03:54:11.422 INFO:tasks.workunit.client.0.smithi110.stdout: client.44741 expiration=0.000000 addr=v1:172.21.15.110:0/3818608367 cookie= 2024-09-14T03:54:11.423 INFO:tasks.workunit.client.0.smithi110.stdout:lock: mylock 2024-09-14T03:54:11.423 INFO:tasks.workunit.client.0.smithi110.stdout: lock_type: shared 2024-09-14T03:54:11.423 INFO:tasks.workunit.client.0.smithi110.stdout: tag: 2024-09-14T03:54:11.423 INFO:tasks.workunit.client.0.smithi110.stdout: lockers: 2024-09-14T03:54:11.424 INFO:tasks.workunit.client.0.smithi110.stdout: client.44741 expiration=0.000000 addr=v1:172.21.15.110:0/3818608367 cookie= 2024-09-14T03:54:11.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:54:11] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.51.0" 2024-09-14T03:54:11.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[88262]: pgmap v1144: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:11.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[88262]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2121143186' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[86423]: pgmap v1144: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[86423]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2121143186' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:11.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:11 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:11 smithi195 ceph-mon[82388]: pgmap v1144: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:11 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:11 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:54:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:11 smithi195 ceph-mon[82388]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T03:54:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:11 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2121143186' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:11.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:11 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:12.402 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestMultiLocking (2882 ms) 2024-09-14T03:54:12.402 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestMeta 2024-09-14T03:54:12.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:12 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:12.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:12 smithi110 ceph-mon[86423]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T03:54:12.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:12.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:12 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:12.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:12 smithi110 ceph-mon[88262]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T03:54:12.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:12.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:12 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:12.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:12 smithi195 ceph-mon[82388]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T03:54:12.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:13.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:13 smithi110 ceph-mon[86423]: pgmap v1147: 137 pgs: 7 creating+peering, 18 unknown, 112 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-14T03:54:13.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:13 smithi110 ceph-mon[86423]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T03:54:13.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:13 smithi110 ceph-mon[88262]: pgmap v1147: 137 pgs: 7 creating+peering, 18 unknown, 112 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-14T03:54:13.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:13 smithi110 ceph-mon[88262]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T03:54:13.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:13 smithi195 ceph-mon[82388]: pgmap v1147: 137 pgs: 7 creating+peering, 18 unknown, 112 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-14T03:54:13.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:13 smithi195 ceph-mon[82388]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T03:54:14.441 INFO:tasks.workunit.client.0.smithi110.stdout:lock: mylock 2024-09-14T03:54:14.441 INFO:tasks.workunit.client.0.smithi110.stdout: lock_type: shared 2024-09-14T03:54:14.441 INFO:tasks.workunit.client.0.smithi110.stdout: tag: 2024-09-14T03:54:14.441 INFO:tasks.workunit.client.0.smithi110.stdout: lockers: 2024-09-14T03:54:14.441 INFO:tasks.workunit.client.0.smithi110.stdout: client.34851 expiration=0.000000 addr=v1:172.21.15.110:0/2534342989 cookie= 2024-09-14T03:54:14.442 INFO:tasks.workunit.client.0.smithi110.stdout:lock: mylock 2024-09-14T03:54:14.442 INFO:tasks.workunit.client.0.smithi110.stdout: lock_type: exclusive 2024-09-14T03:54:14.443 INFO:tasks.workunit.client.0.smithi110.stdout: tag: new_tag 2024-09-14T03:54:14.443 INFO:tasks.workunit.client.0.smithi110.stdout: lockers: 2024-09-14T03:54:14.443 INFO:tasks.workunit.client.0.smithi110.stdout: client.44747 expiration=0.000000 addr=v1:172.21.15.110:0/412927444 cookie= 2024-09-14T03:54:14.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:14 smithi110 ceph-mon[88262]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T03:54:14.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:14 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/412927444' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:14.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:14 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:14.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:14 smithi110 ceph-mon[86423]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T03:54:14.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:14 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/412927444' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:14.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:14 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:14.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:14 smithi195 ceph-mon[82388]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T03:54:14.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:14 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/412927444' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:14.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:14 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:15.415 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestMeta (3013 ms) 2024-09-14T03:54:15.415 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestCookie 2024-09-14T03:54:15.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:15 smithi110 ceph-mon[88262]: pgmap v1150: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:15.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:15 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:15.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:15 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:15.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:15 smithi110 ceph-mon[88262]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T03:54:15.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:15 smithi110 ceph-mon[86423]: pgmap v1150: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:15.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:15 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:15.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:15 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:15.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:15 smithi110 ceph-mon[86423]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T03:54:15.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:15 smithi195 ceph-mon[82388]: pgmap v1150: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:15 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:15 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:15.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:15 smithi195 ceph-mon[82388]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T03:54:16.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:16 smithi110 ceph-mon[88262]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T03:54:16.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:16 smithi110 ceph-mon[86423]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T03:54:16.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:16 smithi195 ceph-mon[82388]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T03:54:17.447 INFO:tasks.workunit.client.0.smithi110.stdout:lock: mylock 2024-09-14T03:54:17.447 INFO:tasks.workunit.client.0.smithi110.stdout: lock_type: exclusive 2024-09-14T03:54:17.447 INFO:tasks.workunit.client.0.smithi110.stdout: tag: 2024-09-14T03:54:17.447 INFO:tasks.workunit.client.0.smithi110.stdout: lockers: 2024-09-14T03:54:17.449 INFO:tasks.workunit.client.0.smithi110.stdout:lock: mylock 2024-09-14T03:54:17.449 INFO:tasks.workunit.client.0.smithi110.stdout: lock_type: shared 2024-09-14T03:54:17.449 INFO:tasks.workunit.client.0.smithi110.stdout: tag: 2024-09-14T03:54:17.449 INFO:tasks.workunit.client.0.smithi110.stdout: lockers: 2024-09-14T03:54:17.449 INFO:tasks.workunit.client.0.smithi110.stdout: client.34857 expiration=0.000000 addr=v1:172.21.15.110:0/461765099 cookie= 2024-09-14T03:54:17.449 INFO:tasks.workunit.client.0.smithi110.stdout: client.34857 expiration=0.000000 addr=v1:172.21.15.110:0/461765099 cookie=new cookie 2024-09-14T03:54:17.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[88262]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:17.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:17.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:54:17.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[88262]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T03:54:17.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/461765099' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:17.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[86423]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:17.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:17.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:54:17.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[86423]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T03:54:17.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:17 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/461765099' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:17.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:17 smithi195 ceph-mon[82388]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:17.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:17 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:17.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:17 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:54:17.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:17 smithi195 ceph-mon[82388]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T03:54:17.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:17 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/461765099' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:18.435 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestCookie (3019 ms) 2024-09-14T03:54:18.435 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestMultipleLocks 2024-09-14T03:54:18.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:18 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/461765099' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:18.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:18 smithi110 ceph-mon[88262]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T03:54:18.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:18 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/461765099' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:18.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:18 smithi110 ceph-mon[86423]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T03:54:18.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:18 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/461765099' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:18.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:18 smithi195 ceph-mon[82388]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T03:54:19.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:19 smithi110 ceph-mon[88262]: pgmap v1156: 137 pgs: 5 creating+peering, 16 unknown, 116 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:19.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:19 smithi110 ceph-mon[88262]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T03:54:19.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:19 smithi110 ceph-mon[86423]: pgmap v1156: 137 pgs: 5 creating+peering, 16 unknown, 116 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:19.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:19 smithi110 ceph-mon[86423]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T03:54:19.708 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:19 smithi195 ceph-mon[82388]: pgmap v1156: 137 pgs: 5 creating+peering, 16 unknown, 116 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:19.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:19 smithi195 ceph-mon[82388]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T03:54:20.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:20 smithi110 ceph-mon[88262]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T03:54:20.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:20 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2762354349' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:20.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:20 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:20.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:20 smithi110 ceph-mon[86423]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T03:54:20.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:20 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2762354349' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:20.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:20 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:20.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:20 smithi195 ceph-mon[82388]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T03:54:20.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:20 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2762354349' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:20.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:20 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:54:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:54:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:21.449 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestMultipleLocks (3013 ms) 2024-09-14T03:54:21.449 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestLockDuration 2024-09-14T03:54:21.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:21 smithi195 ceph-mon[82388]: pgmap v1159: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:21.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:21 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:21.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:21 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:21.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:21 smithi195 ceph-mon[82388]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T03:54:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:54:21] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-09-14T03:54:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-mon[88262]: pgmap v1159: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:21.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-mon[88262]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T03:54:21.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-mon[86423]: pgmap v1159: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:21.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:21.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:21.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:21 smithi110 ceph-mon[86423]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T03:54:22.709 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:22 smithi195 ceph-mon[82388]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T03:54:22.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:22 smithi110 ceph-mon[86423]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T03:54:22.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:22 smithi110 ceph-mon[88262]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T03:54:23.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[88262]: pgmap v1162: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:23.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:23.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:54:23.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[88262]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T03:54:23.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2573438460' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:23.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[86423]: pgmap v1162: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:23.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:23.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:54:23.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[86423]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T03:54:23.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:23 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2573438460' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:23.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:23 smithi195 ceph-mon[82388]: pgmap v1162: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:23 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:23 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:54:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:23 smithi195 ceph-mon[82388]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T03:54:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:23 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2573438460' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:24.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:24 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2573438460' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:24.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:24 smithi110 ceph-mon[88262]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T03:54:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:24 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2573438460' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:24.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:24 smithi110 ceph-mon[86423]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T03:54:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:24 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2573438460' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:24 smithi195 ceph-mon[82388]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T03:54:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:25 smithi110 ceph-mon[88262]: pgmap v1165: 137 pgs: 8 creating+peering, 12 unknown, 117 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:25.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:25 smithi110 ceph-mon[88262]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T03:54:25.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:25 smithi110 ceph-mon[86423]: pgmap v1165: 137 pgs: 8 creating+peering, 12 unknown, 117 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:25.949 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:25 smithi110 ceph-mon[86423]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T03:54:25.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:25 smithi195 ceph-mon[82388]: pgmap v1165: 137 pgs: 8 creating+peering, 12 unknown, 117 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:25.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:25 smithi195 ceph-mon[82388]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T03:54:27.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:27 smithi110 ceph-mon[88262]: pgmap v1167: 137 pgs: 8 creating+peering, 2 unknown, 127 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:27.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:27 smithi110 ceph-mon[86423]: pgmap v1167: 137 pgs: 8 creating+peering, 2 unknown, 127 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:27.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:27.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:27 smithi195 ceph-mon[82388]: pgmap v1167: 137 pgs: 8 creating+peering, 2 unknown, 127 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:27.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:29.483 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestLockDuration (8034 ms) 2024-09-14T03:54:29.483 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestAssertLocked 2024-09-14T03:54:29.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:29 smithi110 ceph-mon[88262]: pgmap v1168: 137 pgs: 137 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:29.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:29 smithi110 ceph-mon[86423]: pgmap v1168: 137 pgs: 137 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:29.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:29 smithi195 ceph-mon[82388]: pgmap v1168: 137 pgs: 137 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:30.488 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:54:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:54:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:30.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:30 smithi110 ceph-mon[88262]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T03:54:30.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:30 smithi110 ceph-mon[86423]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T03:54:30.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:30 smithi195 ceph-mon[82388]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T03:54:31.847 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-mon[86423]: pgmap v1170: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:31.847 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-mon[86423]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T03:54:31.848 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1310257784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:31.848 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:31.848 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:54:31] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-09-14T03:54:31.848 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-mon[88262]: pgmap v1170: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:31.848 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-mon[88262]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T03:54:31.849 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1310257784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:31.849 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:31 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:31.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:31 smithi195 ceph-mon[82388]: pgmap v1170: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:31.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:31 smithi195 ceph-mon[82388]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T03:54:31.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:31 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1310257784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:31.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:31 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:32.509 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestAssertLocked (3026 ms) 2024-09-14T03:54:32.509 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestSetCookie 2024-09-14T03:54:32.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:32 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:32.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:32 smithi110 ceph-mon[86423]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T03:54:32.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:32 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:32.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:32 smithi110 ceph-mon[88262]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T03:54:32.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:32 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:32.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:32 smithi195 ceph-mon[82388]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T03:54:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:33 smithi110 ceph-mon[86423]: pgmap v1173: 137 pgs: 2 creating+peering, 29 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:33.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:33 smithi110 ceph-mon[86423]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T03:54:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:33 smithi110 ceph-mon[88262]: pgmap v1173: 137 pgs: 2 creating+peering, 29 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:33.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:33 smithi110 ceph-mon[88262]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T03:54:33.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:33 smithi195 ceph-mon[82388]: pgmap v1173: 137 pgs: 2 creating+peering, 29 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:33.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:33 smithi195 ceph-mon[82388]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T03:54:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:34 smithi110 ceph-mon[88262]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T03:54:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:34 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/161265058' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:34.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:34 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:34.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:34 smithi110 ceph-mon[86423]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T03:54:34.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:34 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/161265058' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:34.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:34 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:34.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:34 smithi195 ceph-mon[82388]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T03:54:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:34 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/161265058' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:34.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:34 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:35.525 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestSetCookie (3016 ms) 2024-09-14T03:54:35.525 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestRenew 2024-09-14T03:54:35.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:35 smithi110 ceph-mon[88262]: pgmap v1176: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:35.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:35 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:35.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:35 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:35.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:35 smithi110 ceph-mon[88262]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T03:54:35.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:35 smithi110 ceph-mon[86423]: pgmap v1176: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:35.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:35 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:35.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:35 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:35.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:35 smithi110 ceph-mon[86423]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T03:54:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:35 smithi195 ceph-mon[82388]: pgmap v1176: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:35 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:35 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:35.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:35 smithi195 ceph-mon[82388]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T03:54:36.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:36 smithi110 ceph-mon[88262]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T03:54:36.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:36 smithi110 ceph-mon[86423]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T03:54:36.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:36 smithi195 ceph-mon[82388]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T03:54:37.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[88262]: pgmap v1179: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:37.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:37.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:54:37.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[88262]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T03:54:37.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3397734887' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:37.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:37.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[86423]: pgmap v1179: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:37.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:37.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:54:37.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[86423]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T03:54:37.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3397734887' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:37.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:37 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:37.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:37 smithi195 ceph-mon[82388]: pgmap v1179: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:37.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:37 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:37 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:54:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:37 smithi195 ceph-mon[82388]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T03:54:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:37 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3397734887' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:37.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:37 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:38.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:38 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:38.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:38 smithi110 ceph-mon[88262]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T03:54:38.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:38 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:38.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:38 smithi110 ceph-mon[86423]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T03:54:38.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:38 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:38.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:38 smithi195 ceph-mon[82388]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T03:54:39.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:39 smithi110 ceph-mon[88262]: pgmap v1182: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:39.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:39 smithi110 ceph-mon[88262]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T03:54:39.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:39 smithi110 ceph-mon[86423]: pgmap v1182: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:39.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:39 smithi110 ceph-mon[86423]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T03:54:39.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:39 smithi195 ceph-mon[82388]: pgmap v1182: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:39.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:39 smithi195 ceph-mon[82388]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T03:54:40.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:54:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:54:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:41 smithi110 ceph-mon[88262]: pgmap v1184: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:41.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:41 smithi110 ceph-mon[86423]: pgmap v1184: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:54:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:54:41] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-09-14T03:54:41.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:41 smithi195 ceph-mon[82388]: pgmap v1184: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:42.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:42.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:42.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:43.860 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:43 smithi195 ceph-mon[82388]: pgmap v1185: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:43.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:43 smithi110 ceph-mon[88262]: pgmap v1185: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:43.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:43 smithi110 ceph-mon[86423]: pgmap v1185: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:45.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:45 smithi110 ceph-mon[88262]: pgmap v1186: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:45.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:45 smithi110 ceph-mon[86423]: pgmap v1186: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:45.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:45 smithi195 ceph-mon[82388]: pgmap v1186: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:47.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:47 smithi110 ceph-mon[88262]: pgmap v1187: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:47.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:47 smithi110 ceph-mon[86423]: pgmap v1187: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:47.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:47 smithi195 ceph-mon[82388]: pgmap v1187: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:49.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:49 smithi110 ceph-mon[88262]: pgmap v1188: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:49.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:49 smithi110 ceph-mon[86423]: pgmap v1188: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:49.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:49 smithi195 ceph-mon[82388]: pgmap v1188: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:54:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:54:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:51.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:51 smithi110 ceph-mon[88262]: pgmap v1189: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 359 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T03:54:51.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:51 smithi110 ceph-mon[86423]: pgmap v1189: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 359 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T03:54:51.947 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:54:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:54:51] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.51.0" 2024-09-14T03:54:51.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:51 smithi195 ceph-mon[82388]: pgmap v1189: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 359 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T03:54:53.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:53 smithi110 ceph-mon[88262]: pgmap v1190: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:53.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:53 smithi110 ceph-mon[86423]: pgmap v1190: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:53.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:53 smithi195 ceph-mon[82388]: pgmap v1190: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:55.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:55 smithi110 ceph-mon[88262]: pgmap v1191: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:55.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:55 smithi110 ceph-mon[86423]: pgmap v1191: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:55.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:55 smithi195 ceph-mon[82388]: pgmap v1191: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:56.579 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestRenew (21055 ms) 2024-09-14T03:54:56.579 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2024-09-14T03:54:57.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:57 smithi110 ceph-mon[88262]: pgmap v1192: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:57.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:57 smithi110 ceph-mon[88262]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T03:54:57.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:57.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:57 smithi110 ceph-mon[86423]: pgmap v1192: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:57.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:57 smithi110 ceph-mon[86423]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T03:54:57.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:57.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:57 smithi195 ceph-mon[82388]: pgmap v1192: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:54:57.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:57 smithi195 ceph-mon[82388]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T03:54:57.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:58.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:58 smithi110 ceph-mon[88262]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T03:54:58.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:58 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2878163378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:58.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:58 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:58.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:58 smithi110 ceph-mon[86423]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T03:54:58.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:58 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2878163378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:58.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:58 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:58.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:58 smithi195 ceph-mon[82388]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T03:54:58.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:58 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2878163378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:58.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:58 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:54:59.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:59 smithi110 ceph-mon[88262]: pgmap v1195: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:59.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:59 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:59.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:59 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:59.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:54:59 smithi110 ceph-mon[88262]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T03:54:59.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:59 smithi110 ceph-mon[86423]: pgmap v1195: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:59.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:59 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:59.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:59 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:59.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:54:59 smithi110 ceph-mon[86423]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T03:54:59.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:59 smithi195 ceph-mon[82388]: pgmap v1195: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:54:59.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:59 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:59.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:59 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:59.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:54:59 smithi195 ceph-mon[82388]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T03:55:00.592 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:55:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:55:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:00.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:00 smithi110 ceph-mon[88262]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T03:55:00.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:00 smithi110 ceph-mon[86423]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T03:55:00.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:00 smithi195 ceph-mon[82388]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T03:55:01.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:01 smithi110 ceph-mon[88262]: pgmap v1198: 137 pgs: 9 creating+peering, 8 unknown, 120 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:01.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:01 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:01.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:01 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:55:01.848 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:55:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:55:01] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-14T03:55:01.848 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:01 smithi110 ceph-mon[86423]: pgmap v1198: 137 pgs: 9 creating+peering, 8 unknown, 120 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:01.848 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:01 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:01.848 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:01 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:55:01.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:01 smithi195 ceph-mon[82388]: pgmap v1198: 137 pgs: 9 creating+peering, 8 unknown, 120 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:01.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:01 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:01.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:01 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:55:03.604 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (7024 ms) 2024-09-14T03:55:03.604 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2024-09-14T03:55:03.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:03 smithi110 ceph-mon[88262]: pgmap v1199: 137 pgs: 9 creating+peering, 128 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 191 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T03:55:03.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:03 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:03.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:03 smithi110 ceph-mon[86423]: pgmap v1199: 137 pgs: 9 creating+peering, 128 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 191 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T03:55:03.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:03 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:03.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:03 smithi195 ceph-mon[82388]: pgmap v1199: 137 pgs: 9 creating+peering, 128 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 191 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T03:55:03.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:03 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:04.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:04 smithi110 ceph-mon[86423]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T03:55:04.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:04 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:04.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:04 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:04.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:04 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:55:04.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:04 smithi110 ceph-mon[88262]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T03:55:04.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:04 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:04.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:04 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:04.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:04 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:55:04.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:04 smithi195 ceph-mon[82388]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T03:55:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:04 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:04 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:04.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:04 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:55:05.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:05 smithi110 ceph-mon[88262]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:05.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:05 smithi110 ceph-mon[88262]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T03:55:05.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:05 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3063870675' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:05.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:05 smithi110 ceph-mon[86423]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:05.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:05 smithi110 ceph-mon[86423]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T03:55:05.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:05 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3063870675' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:05.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:05 smithi195 ceph-mon[82388]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:05 smithi195 ceph-mon[82388]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T03:55:05.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:05 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3063870675' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:06.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:06 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3063870675' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:06.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:06 smithi110 ceph-mon[86423]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T03:55:06.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:06 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3063870675' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:06.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:06 smithi110 ceph-mon[88262]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T03:55:06.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:06 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3063870675' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:06.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:06 smithi195 ceph-mon[82388]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T03:55:07.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:07 smithi110 ceph-mon[86423]: pgmap v1204: 137 pgs: 5 creating+peering, 21 unknown, 111 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:07.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:07 smithi110 ceph-mon[86423]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T03:55:07.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:07 smithi110 ceph-mon[88262]: pgmap v1204: 137 pgs: 5 creating+peering, 21 unknown, 111 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:07.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:07 smithi110 ceph-mon[88262]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T03:55:07.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:07 smithi195 ceph-mon[82388]: pgmap v1204: 137 pgs: 5 creating+peering, 21 unknown, 111 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:07.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:07 smithi195 ceph-mon[82388]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T03:55:09.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:09 smithi110 ceph-mon[88262]: pgmap v1206: 137 pgs: 5 creating+peering, 9 unknown, 123 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:09.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:09 smithi110 ceph-mon[86423]: pgmap v1206: 137 pgs: 5 creating+peering, 9 unknown, 123 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:09.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:09 smithi195 ceph-mon[82388]: pgmap v1206: 137 pgs: 5 creating+peering, 9 unknown, 123 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:10.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:55:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:55:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:11.639 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (8035 ms) 2024-09-14T03:55:11.639 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2024-09-14T03:55:11.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:11 smithi110 ceph-mon[88262]: pgmap v1207: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:11.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:11 smithi110 ceph-mon[86423]: pgmap v1207: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:11.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:55:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:55:11] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-14T03:55:11.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:11 smithi195 ceph-mon[82388]: pgmap v1207: 137 pgs: 137 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:12.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:12 smithi110 ceph-mon[88262]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T03:55:12.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:12.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:12 smithi110 ceph-mon[86423]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T03:55:12.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:12.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:12 smithi195 ceph-mon[82388]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T03:55:12.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:13.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:13 smithi110 ceph-mon[86423]: pgmap v1209: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:13.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:13 smithi110 ceph-mon[86423]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T03:55:13.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:13 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3989982729' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:13.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:13 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:13 smithi110 ceph-mon[88262]: pgmap v1209: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:13 smithi110 ceph-mon[88262]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T03:55:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:13 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3989982729' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:13.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:13 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:13.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:13 smithi195 ceph-mon[82388]: pgmap v1209: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:13 smithi195 ceph-mon[82388]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T03:55:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:13 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3989982729' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:13.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:13 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:14.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:14 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:14.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:14 smithi110 ceph-mon[86423]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T03:55:14.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:14 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:14.948 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:14 smithi110 ceph-mon[88262]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T03:55:14.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:14 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-117875-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:14.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:14 smithi195 ceph-mon[82388]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T03:55:15.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:15 smithi110 ceph-mon[88262]: pgmap v1212: 137 pgs: 13 creating+peering, 14 unknown, 110 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:15.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:15 smithi110 ceph-mon[88262]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T03:55:15.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:15 smithi110 ceph-mon[86423]: pgmap v1212: 137 pgs: 13 creating+peering, 14 unknown, 110 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:15.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:15 smithi110 ceph-mon[86423]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T03:55:15.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:15 smithi195 ceph-mon[82388]: pgmap v1212: 137 pgs: 13 creating+peering, 14 unknown, 110 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:15.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:15 smithi195 ceph-mon[82388]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T03:55:17.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:17 smithi110 ceph-mon[88262]: pgmap v1214: 137 pgs: 13 creating+peering, 4 unknown, 120 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:17.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:17 smithi110 ceph-mon[86423]: pgmap v1214: 137 pgs: 13 creating+peering, 4 unknown, 120 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:17.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:17 smithi195 ceph-mon[82388]: pgmap v1214: 137 pgs: 13 creating+peering, 4 unknown, 120 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:19.673 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (8035 ms) 2024-09-14T03:55:19.674 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 11 tests from ClsLock (70154 ms total) 2024-09-14T03:55:19.674 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:55:19.674 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:55:19.674 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 11 tests from 1 test suite ran. (70154 ms total) 2024-09-14T03:55:19.674 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 11 tests. 2024-09-14T03:55:19.676 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:55:19.676 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:55:19.745 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2024-09-14T03:55:19.745 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2024-09-14T03:55:19.814 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:55:19.815 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-14T03:55:19.815 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:55:19.815 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 4 tests from cls_log 2024-09-14T03:55:19.815 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_log.test_log_add_same_time 2024-09-14T03:55:19.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:19 smithi110 ceph-mon[88262]: pgmap v1215: 137 pgs: 137 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:19.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:19 smithi110 ceph-mon[86423]: pgmap v1215: 137 pgs: 137 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:19.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:19 smithi195 ceph-mon[82388]: pgmap v1215: 137 pgs: 137 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T03:55:20.671 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:55:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:55:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:20.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:20 smithi110 ceph-mon[88262]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T03:55:20.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:20 smithi110 ceph-mon[86423]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T03:55:20.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:20 smithi195 ceph-mon[82388]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T03:55:21.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:55:21] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.51.0" 2024-09-14T03:55:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-mon[86423]: pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-mon[86423]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T03:55:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4139084869' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:21.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:21.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-mon[88262]: pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:21.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-mon[88262]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T03:55:21.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4139084869' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:21.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:21 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:21.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:21 smithi195 ceph-mon[82388]: pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:21.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:21 smithi195 ceph-mon[82388]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T03:55:21.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:21 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4139084869' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:21.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:21 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:22.690 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_log.test_log_add_same_time (2875 ms) 2024-09-14T03:55:22.690 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_log.test_log_add_different_time 2024-09-14T03:55:22.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:22 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:22.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:22 smithi110 ceph-mon[88262]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T03:55:22.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:22 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:22.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:22 smithi110 ceph-mon[86423]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T03:55:22.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:22 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:22.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:22 smithi195 ceph-mon[82388]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T03:55:23.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:23 smithi110 ceph-mon[88262]: pgmap v1220: 137 pgs: 6 creating+peering, 25 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:23.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:23 smithi110 ceph-mon[88262]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T03:55:23.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:23 smithi110 ceph-mon[86423]: pgmap v1220: 137 pgs: 6 creating+peering, 25 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:23.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:23 smithi110 ceph-mon[86423]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T03:55:23.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:23 smithi195 ceph-mon[82388]: pgmap v1220: 137 pgs: 6 creating+peering, 25 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:23.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:23 smithi195 ceph-mon[82388]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T03:55:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:24 smithi195 ceph-mon[82388]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T03:55:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:24 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3074423911' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:24.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:24 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:24 smithi110 ceph-mon[88262]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T03:55:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:24 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3074423911' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:25.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:24 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:25.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:24 smithi110 ceph-mon[86423]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T03:55:25.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:24 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3074423911' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:25.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:24 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:25.706 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_log.test_log_add_different_time (3017 ms) 2024-09-14T03:55:25.707 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_log.trim_by_time 2024-09-14T03:55:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:25 smithi195 ceph-mon[82388]: pgmap v1223: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:25 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:25 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:25.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:25 smithi195 ceph-mon[82388]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T03:55:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:25 smithi110 ceph-mon[88262]: pgmap v1223: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:25 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:25 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:26.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:25 smithi110 ceph-mon[88262]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T03:55:26.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:25 smithi110 ceph-mon[86423]: pgmap v1223: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:26.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:25 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:26.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:25 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:26.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:25 smithi110 ceph-mon[86423]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T03:55:27.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:26 smithi110 ceph-mon[88262]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T03:55:27.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:26 smithi110 ceph-mon[86423]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T03:55:27.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:26 smithi195 ceph-mon[82388]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T03:55:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[88262]: pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:55:28.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[88262]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2488388002' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[86423]: pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[86423]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2488388002' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:28.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:27 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:27 smithi195 ceph-mon[82388]: pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:27 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:27 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:55:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:27 smithi195 ceph-mon[82388]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T03:55:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:27 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2488388002' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:27 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:28.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:27 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:28.732 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_log.trim_by_time (3025 ms) 2024-09-14T03:55:28.732 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] cls_log.trim_by_marker 2024-09-14T03:55:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:28 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:29.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:28 smithi110 ceph-mon[88262]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T03:55:29.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:28 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:29.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:28 smithi110 ceph-mon[86423]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T03:55:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:28 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:29.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:28 smithi195 ceph-mon[82388]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T03:55:30.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:29 smithi110 ceph-mon[88262]: pgmap v1229: 137 pgs: 6 creating+peering, 16 unknown, 115 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:30.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:29 smithi110 ceph-mon[88262]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T03:55:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:29 smithi110 ceph-mon[86423]: pgmap v1229: 137 pgs: 6 creating+peering, 16 unknown, 115 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:30.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:29 smithi110 ceph-mon[86423]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T03:55:30.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:29 smithi195 ceph-mon[82388]: pgmap v1229: 137 pgs: 6 creating+peering, 16 unknown, 115 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:30.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:29 smithi195 ceph-mon[82388]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T03:55:30.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:55:30 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:55:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:31.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:30 smithi110 ceph-mon[88262]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T03:55:31.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:30 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/260004123' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:31.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:30 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:31.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:30 smithi110 ceph-mon[86423]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T03:55:31.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:30 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/260004123' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:31.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:30 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:31.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:30 smithi195 ceph-mon[82388]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T03:55:31.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:30 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/260004123' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:31.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:30 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:31.753 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] cls_log.trim_by_marker (3021 ms) 2024-09-14T03:55:31.753 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 4 tests from cls_log (11938 ms total) 2024-09-14T03:55:31.753 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:55:31.753 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:55:31.753 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 4 tests from 1 test suite ran. (11938 ms total) 2024-09-14T03:55:31.753 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 4 tests. 2024-09-14T03:55:31.755 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:55:31.755 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:55:31.820 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:55:31] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.51.0" 2024-09-14T03:55:31.821 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-mon[88262]: pgmap v1232: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:31.821 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:31.821 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-mon[88262]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T03:55:31.821 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:31.821 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-mon[86423]: pgmap v1232: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:31.821 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:31.821 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-mon[86423]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T03:55:31.821 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:31 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:31.824 INFO:tasks.workunit:Running workunit cls/test_cls_numops.sh... 2024-09-14T03:55:31.824 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_numops.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_numops.sh 2024-09-14T03:55:31.893 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:55:31.893 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-14T03:55:31.893 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:55:31.893 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 4 tests from ClsNumOps 2024-09-14T03:55:31.893 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsNumOps.Add 2024-09-14T03:55:32.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:31 smithi195 ceph-mon[82388]: pgmap v1232: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:31 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118212-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:31 smithi195 ceph-mon[82388]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T03:55:32.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:31 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:33.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:32 smithi110 ceph-mon[86423]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T03:55:33.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:32 smithi110 ceph-mon[88262]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T03:55:33.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:32 smithi195 ceph-mon[82388]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T03:55:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[86423]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:34.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[86423]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4018337690' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[88262]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[88262]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4018337690' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:34.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:33 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:34.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:33 smithi195 ceph-mon[82388]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:33 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:33 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:55:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:33 smithi195 ceph-mon[82388]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T03:55:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:33 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4018337690' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:34.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:33 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:34.763 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsNumOps.Add (2869 ms) 2024-09-14T03:55:34.763 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsNumOps.Sub 2024-09-14T03:55:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:34 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:34 smithi110 ceph-mon[86423]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T03:55:35.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:34 smithi110 ceph-mon[86423]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T03:55:35.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:34 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:35.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:34 smithi110 ceph-mon[88262]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T03:55:35.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:34 smithi110 ceph-mon[88262]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T03:55:35.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:34 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:34 smithi195 ceph-mon[82388]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T03:55:35.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:34 smithi195 ceph-mon[82388]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T03:55:36.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:35 smithi110 ceph-mon[88262]: pgmap v1238: 137 pgs: 11 creating+peering, 20 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:36.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:35 smithi110 ceph-mon[86423]: pgmap v1238: 137 pgs: 11 creating+peering, 20 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:36.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:35 smithi195 ceph-mon[82388]: pgmap v1238: 137 pgs: 11 creating+peering, 20 unknown, 106 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:36 smithi110 ceph-mon[88262]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T03:55:37.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:36 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1138589314' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:37.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:36 smithi110 ceph-mon[86423]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T03:55:37.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:36 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1138589314' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:36 smithi195 ceph-mon[82388]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T03:55:37.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:36 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1138589314' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:37.790 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsNumOps.Sub (3027 ms) 2024-09-14T03:55:37.790 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsNumOps.Mul 2024-09-14T03:55:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:37 smithi110 ceph-mon[88262]: pgmap v1241: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:37 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:37 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1138589314' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:38.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:37 smithi110 ceph-mon[88262]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T03:55:38.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:37 smithi110 ceph-mon[86423]: pgmap v1241: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:38.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:37 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:38.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:37 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1138589314' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:38.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:37 smithi110 ceph-mon[86423]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T03:55:38.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:37 smithi195 ceph-mon[82388]: pgmap v1241: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:37 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:37 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1138589314' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:38.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:37 smithi195 ceph-mon[82388]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T03:55:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:38 smithi110 ceph-mon[88262]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T03:55:39.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:38 smithi110 ceph-mon[86423]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T03:55:39.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:38 smithi195 ceph-mon[82388]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T03:55:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[88262]: pgmap v1244: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:55:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[88262]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T03:55:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3809201057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:40.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[86423]: pgmap v1244: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:55:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[86423]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T03:55:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3809201057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:40.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:39 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:40.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:39 smithi195 ceph-mon[82388]: pgmap v1244: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:39 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:39 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:55:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:39 smithi195 ceph-mon[82388]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T03:55:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:39 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3809201057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:40.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:39 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:40.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:55:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:55:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:40.814 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsNumOps.Mul (3025 ms) 2024-09-14T03:55:40.814 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] ClsNumOps.Div 2024-09-14T03:55:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:40 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:41.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:40 smithi110 ceph-mon[88262]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T03:55:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:40 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:41.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:40 smithi110 ceph-mon[86423]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T03:55:41.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:40 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:41.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:40 smithi195 ceph-mon[82388]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T03:55:41.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:55:41 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:55:41] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.51.0" 2024-09-14T03:55:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:41 smithi110 ceph-mon[88262]: pgmap v1247: 137 pgs: 6 creating+peering, 23 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:41.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:41 smithi110 ceph-mon[88262]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T03:55:41.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:41 smithi110 ceph-mon[86423]: pgmap v1247: 137 pgs: 6 creating+peering, 23 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:41.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:41 smithi110 ceph-mon[86423]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T03:55:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:41 smithi195 ceph-mon[82388]: pgmap v1247: 137 pgs: 6 creating+peering, 23 unknown, 108 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:42.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:41 smithi195 ceph-mon[82388]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T03:55:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:42 smithi195 ceph-mon[82388]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T03:55:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:42 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/268403168' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:43.110 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:42 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:42 smithi110 ceph-mon[88262]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T03:55:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:42 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/268403168' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:43.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:42 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:42 smithi110 ceph-mon[86423]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T03:55:43.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:42 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/268403168' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:43.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:42 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:43.831 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] ClsNumOps.Div (3016 ms) 2024-09-14T03:55:43.831 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 4 tests from ClsNumOps (11937 ms total) 2024-09-14T03:55:43.831 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:55:43.831 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:55:43.831 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 4 tests from 1 test suite ran. (11937 ms total) 2024-09-14T03:55:43.831 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 4 tests. 2024-09-14T03:55:43.833 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:55:43.833 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:55:43.902 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-14T03:55:43.902 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-14T03:55:43.972 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2024-09-14T03:55:43.972 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-14T03:55:43.972 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2024-09-14T03:55:43.972 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 70 tests from TestClsRbd 2024-09-14T03:55:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:43 smithi110 ceph-mon[88262]: pgmap v1250: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:43 smithi110 ceph-mon[88262]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:43 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/268403168' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:44.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:43 smithi110 ceph-mon[88262]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T03:55:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:43 smithi110 ceph-mon[86423]: pgmap v1250: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:43 smithi110 ceph-mon[86423]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:43 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/268403168' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:44.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:43 smithi110 ceph-mon[86423]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T03:55:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:43 smithi195 ceph-mon[82388]: pgmap v1250: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:43 smithi195 ceph-mon[82388]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:43 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/268403168' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118318-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:44.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:43 smithi195 ceph-mon[82388]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T03:55:45.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:44 smithi110 ceph-mon[88262]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T03:55:45.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:44 smithi110 ceph-mon[86423]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T03:55:45.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:44 smithi195 ceph-mon[82388]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T03:55:45.839 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-14T03:55:45.855 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_all_features (16 ms) 2024-09-14T03:55:45.855 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.copyup 2024-09-14T03:55:46.033 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.copyup (178 ms) 2024-09-14T03:55:46.034 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-14T03:55:46.045 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.sparse_copyup (11 ms) 2024-09-14T03:55:46.045 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-14T03:55:46.051 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-09-14T03:55:46.051 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-14T03:55:46.061 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2024-09-14T03:55:46.062 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-14T03:55:46.089 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.directory_methods (28 ms) 2024-09-14T03:55:46.089 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.create 2024-09-14T03:55:46.102 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.create (12 ms) 2024-09-14T03:55:46.102 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_features 2024-09-14T03:55:46.103 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-14T03:55:46.103 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-14T03:55:46.104 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-09-14T03:55:46.105 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-14T03:55:46.106 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-14T03:55:46.106 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-14T03:55:46.108 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-09-14T03:55:46.108 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-14T03:55:46.109 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-14T03:55:46.109 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-14T03:55:46.113 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-09-14T03:55:46.113 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_size 2024-09-14T03:55:46.116 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2024-09-14T03:55:46.116 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.set_size 2024-09-14T03:55:46.120 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-09-14T03:55:46.121 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-14T03:55:46.138 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.protection_status (18 ms) 2024-09-14T03:55:46.138 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-14T03:55:46.146 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2024-09-14T03:55:46.147 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-14T03:55:46.185 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.parents_v1 (38 ms) 2024-09-14T03:55:46.185 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-14T03:55:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[88262]: pgmap v1253: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[88262]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:55:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[88262]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T03:55:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2618024938' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:46.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[86423]: pgmap v1253: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:46.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[86423]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:55:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[86423]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T03:55:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2618024938' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:46.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:45 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:45 smithi195 ceph-mon[82388]: pgmap v1253: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:45 smithi195 ceph-mon[82388]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:55:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:45 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:55:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:45 smithi195 ceph-mon[82388]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T03:55:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:45 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2618024938' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:46.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:45 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:55:46.211 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.parents_v2 (27 ms) 2024-09-14T03:55:46.211 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-14T03:55:46.231 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.snapshots (20 ms) 2024-09-14T03:55:46.231 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-14T03:55:46.235 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.snapid_race (3 ms) 2024-09-14T03:55:46.235 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-14T03:55:46.243 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-09-14T03:55:46.243 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-14T03:55:46.244 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2024-09-14T03:55:46.244 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-14T03:55:46.249 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_resize (4 ms) 2024-09-14T03:55:46.249 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-14T03:55:46.252 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2024-09-14T03:55:46.253 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-14T03:55:46.253 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-14T03:55:46.253 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-14T03:55:46.257 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2024-09-14T03:55:46.257 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-14T03:55:46.260 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2024-09-14T03:55:46.260 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.flags 2024-09-14T03:55:46.265 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.flags (4 ms) 2024-09-14T03:55:46.265 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.metadata 2024-09-14T03:55:46.286 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.metadata (22 ms) 2024-09-14T03:55:46.287 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.set_features 2024-09-14T03:55:46.294 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2024-09-14T03:55:46.294 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror 2024-09-14T03:55:46.324 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror (31 ms) 2024-09-14T03:55:46.325 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-14T03:55:46.337 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_image (13 ms) 2024-09-14T03:55:46.337 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-14T03:55:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:46 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:47.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:46 smithi110 ceph-mon[88262]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T03:55:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:46 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:47.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:46 smithi110 ceph-mon[86423]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T03:55:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:46 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-118424-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:47.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:46 smithi195 ceph-mon[82388]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T03:55:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:47 smithi110 ceph-mon[88262]: pgmap v1256: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:48.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:47 smithi110 ceph-mon[88262]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T03:55:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:47 smithi110 ceph-mon[86423]: pgmap v1256: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:48.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:47 smithi110 ceph-mon[86423]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T03:55:48.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:47 smithi195 ceph-mon[82388]: pgmap v1256: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:48.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:47 smithi195 ceph-mon[82388]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T03:55:48.466 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_image_status (2129 ms) 2024-09-14T03:55:48.466 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-14T03:55:48.543 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_image_map (77 ms) 2024-09-14T03:55:48.544 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-14T03:55:48.551 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_instances (7 ms) 2024-09-14T03:55:48.551 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-14T03:55:48.561 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2024-09-14T03:55:48.561 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-14T03:55:48.563 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-14T03:55:48.563 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-14T03:55:48.565 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-14T03:55:48.565 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-14T03:55:48.568 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-14T03:55:48.568 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-14T03:55:48.574 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2024-09-14T03:55:48.574 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-14T03:55:48.576 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-14T03:55:48.577 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-14T03:55:48.580 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-14T03:55:48.580 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-14T03:55:48.582 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-14T03:55:48.582 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-14T03:55:48.585 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-14T03:55:48.585 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-14T03:55:48.588 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-14T03:55:48.588 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-14T03:55:48.592 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-14T03:55:48.592 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-14T03:55:48.594 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-09-14T03:55:48.594 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-14T03:55:48.597 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-14T03:55:48.597 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-14T03:55:48.599 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-14T03:55:48.599 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-14T03:55:48.601 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-14T03:55:48.601 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-14T03:55:48.603 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-14T03:55:48.603 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-14T03:55:48.605 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2024-09-14T03:55:48.605 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-14T03:55:48.608 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-09-14T03:55:48.608 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-14T03:55:48.610 INFO:tasks.workunit.client.0.smithi110.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1629-geb16b97b/rpm/el9/BUILD/ceph-17.2.7-1629-geb16b97b/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-14T03:55:48.610 INFO:tasks.workunit.client.0.smithi110.stdout:Expected equality of these values: 2024-09-14T03:55:48.610 INFO:tasks.workunit.client.0.smithi110.stdout: 1U 2024-09-14T03:55:48.610 INFO:tasks.workunit.client.0.smithi110.stdout: Which is: 1 2024-09-14T03:55:48.610 INFO:tasks.workunit.client.0.smithi110.stdout: keys.size() 2024-09-14T03:55:48.610 INFO:tasks.workunit.client.0.smithi110.stdout: Which is: 3 2024-09-14T03:55:48.610 INFO:tasks.workunit.client.0.smithi110.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-14T03:55:48.610 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-14T03:55:48.613 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-14T03:55:48.613 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-14T03:55:48.627 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (14 ms) 2024-09-14T03:55:48.627 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-14T03:55:48.828 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (201 ms) 2024-09-14T03:55:48.828 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-14T03:55:48.830 INFO:tasks.workunit.client.0.smithi110.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1629-geb16b97b/rpm/el9/BUILD/ceph-17.2.7-1629-geb16b97b/src/test/cls_rbd/test_cls_rbd.cc:2792: Failure 2024-09-14T03:55:48.830 INFO:tasks.workunit.client.0.smithi110.stdout:Expected equality of these values: 2024-09-14T03:55:48.830 INFO:tasks.workunit.client.0.smithi110.stdout: 1U 2024-09-14T03:55:48.830 INFO:tasks.workunit.client.0.smithi110.stdout: Which is: 1 2024-09-14T03:55:48.831 INFO:tasks.workunit.client.0.smithi110.stdout: keys.size() 2024-09-14T03:55:48.831 INFO:tasks.workunit.client.0.smithi110.stdout: Which is: 3 2024-09-14T03:55:48.831 INFO:tasks.workunit.client.0.smithi110.stdout:[ FAILED ] TestClsRbd.group_snap_remove (2 ms) 2024-09-14T03:55:48.831 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-14T03:55:48.832 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-14T03:55:48.833 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-14T03:55:48.839 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.trash_methods (6 ms) 2024-09-14T03:55:48.839 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.op_features 2024-09-14T03:55:48.845 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2024-09-14T03:55:48.846 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-14T03:55:48.865 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.clone_parent (19 ms) 2024-09-14T03:55:48.865 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-14T03:55:48.872 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-14T03:55:48.872 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-14T03:55:48.882 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2024-09-14T03:55:48.882 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-14T03:55:48.889 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-09-14T03:55:48.889 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.migration 2024-09-14T03:55:48.898 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-09-14T03:55:48.899 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-14T03:55:48.906 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.migration_v1 (8 ms) 2024-09-14T03:55:48.906 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-14T03:55:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:49 smithi110 ceph-mon[88262]: pgmap v1258: 137 pgs: 8 creating+peering, 13 unknown, 116 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 8.3 KiB/s wr, 34 op/s 2024-09-14T03:55:50.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:49 smithi110 ceph-mon[86423]: pgmap v1258: 137 pgs: 8 creating+peering, 13 unknown, 116 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 8.3 KiB/s wr, 34 op/s 2024-09-14T03:55:50.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:49 smithi195 ceph-mon[82388]: pgmap v1258: 137 pgs: 8 creating+peering, 13 unknown, 116 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 8.3 KiB/s wr, 34 op/s 2024-09-14T03:55:50.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:55:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:55:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:50.859 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1952 ms) 2024-09-14T03:55:50.859 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-14T03:55:50.900 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.sparsify (42 ms) 2024-09-14T03:55:51.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:50 smithi110 ceph-mon[86423]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T03:55:51.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:50 smithi110 ceph-mon[88262]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T03:55:51.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:50 smithi195 ceph-mon[82388]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T03:55:51.861 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:55:51 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:55:51] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-09-14T03:55:51.866 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 70 tests from TestClsRbd (5061 ms total) 2024-09-14T03:55:51.866 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:55:51.866 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2024-09-14T03:55:51.867 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 70 tests from 1 test suite ran. (7895 ms total) 2024-09-14T03:55:51.867 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 68 tests. 2024-09-14T03:55:51.867 INFO:tasks.workunit.client.0.smithi110.stdout:[ FAILED ] 2 tests, listed below: 2024-09-14T03:55:51.867 INFO:tasks.workunit.client.0.smithi110.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-14T03:55:51.867 INFO:tasks.workunit.client.0.smithi110.stdout:[ FAILED ] TestClsRbd.group_snap_remove 2024-09-14T03:55:51.867 INFO:tasks.workunit.client.0.smithi110.stdout: 2024-09-14T03:55:51.867 INFO:tasks.workunit.client.0.smithi110.stdout: 2 FAILED TESTS 2024-09-14T03:55:51.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:55:51.869 INFO:tasks.workunit:Stopping ['cls'] on client.0... 2024-09-14T03:55:51.869 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-14T03:55:52.122 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:51 smithi110 ceph-mon[88262]: pgmap v1260: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.98k op/s 2024-09-14T03:55:52.122 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:51 smithi110 ceph-mon[88262]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T03:55:52.122 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:51 smithi110 ceph-mon[86423]: pgmap v1260: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.98k op/s 2024-09-14T03:55:52.122 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:51 smithi110 ceph-mon[86423]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T03:55:52.124 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi110 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-14T03:55:52.357 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=2efbdfcab6e64117b08c26dc9412f8bf Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi110 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-14T03:55:52.361 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-14T03:55:52.372 INFO:tasks.cephadm:Teardown begin 2024-09-14T03:55:52.372 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-09-14T03:55:52.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:55:52.378 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:51 smithi195 ceph-mon[82388]: pgmap v1260: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.98k op/s 2024-09-14T03:55:52.378 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:51 smithi195 ceph-mon[82388]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T03:55:52.619 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:55:53.020 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:52 smithi110 ceph-mon[88262]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T03:55:53.021 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:52 smithi110 ceph-mon[86423]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T03:55:53.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:52 smithi195 ceph-mon[82388]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T03:55:54.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:53 smithi110 ceph-mon[88262]: pgmap v1263: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:54.130 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:53 smithi110 ceph-mon[86423]: pgmap v1263: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:54.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:53 smithi195 ceph-mon[82388]: pgmap v1263: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:54.634 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:55:54.634 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":19,"btime":"2024-09-14T03:46:28:760751+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54295,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.195:6833/2567437763","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6832","nonce":2567437763},{"type":"v1","addr":"172.21.15.195:6833","nonce":2567437763}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":19}],"filesystems":[{"mdsmap":{"epoch":18,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:34:04.160374+0000","modified":"2024-09-14T03:46:19.635975+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":576,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":44435},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44435":{"gid":44435,"name":"a","rank":0,"incarnation":15,"state":"up:active","state_seq":6,"addr":"172.21.15.110:6835/1777878812","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6834","nonce":1777878812},{"type":"v1","addr":"172.21.15.110:6835","nonce":1777878812}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":44435,"qdb_cluster":[44435]},"id":1}]} 2024-09-14T03:55:54.635 INFO:teuthology.orchestra.run.smithi110.stderr:dumped fsmap epoch 19 2024-09-14T03:55:55.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:54 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/513603028' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T03:55:55.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:54 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/513603028' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T03:55:55.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:54 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/513603028' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T03:55:55.227 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:55:55.482 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:55:56.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:55 smithi110 ceph-mon[88262]: pgmap v1264: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:56.196 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:55 smithi110 ceph-mon[86423]: pgmap v1264: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:56.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:55 smithi195 ceph-mon[82388]: pgmap v1264: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:57.328 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:55:57.329 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":19,"btime":"2024-09-14T03:46:28:760751+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54295,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.195:6833/2567437763","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6832","nonce":2567437763},{"type":"v1","addr":"172.21.15.195:6833","nonce":2567437763}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":19}],"filesystems":[{"mdsmap":{"epoch":18,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:34:04.160374+0000","modified":"2024-09-14T03:46:19.635975+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":576,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":44435},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44435":{"gid":44435,"name":"a","rank":0,"incarnation":15,"state":"up:active","state_seq":6,"addr":"172.21.15.110:6835/1777878812","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6834","nonce":1777878812},{"type":"v1","addr":"172.21.15.110:6835","nonce":1777878812}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":44435,"qdb_cluster":[44435]},"id":1}]} 2024-09-14T03:55:57.329 INFO:teuthology.orchestra.run.smithi110.stderr:dumped fsmap epoch 19 2024-09-14T03:55:57.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:57 smithi110 ceph-mon[88262]: pgmap v1265: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:57.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:57 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:57.946 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:57 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3922683181' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T03:55:57.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:57 smithi110 ceph-mon[86423]: pgmap v1265: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:57.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:57 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:57.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:57 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3922683181' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T03:55:57.994 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:55:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:57 smithi195 ceph-mon[82388]: pgmap v1265: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:55:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:57 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:58.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:57 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3922683181' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T03:55:58.236 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:00.109 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-14T03:56:00.110 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:55:59 smithi110 ceph-mon[86423]: pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:00.110 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:55:59 smithi110 ceph-mon[88262]: pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:00.111 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1059,"fsid":"5517b266-7249-11ef-bceb-c7b262605968","created":"2024-09-14T03:28:39.369757+0000","modified":"2024-09-14T03:55:51.851871+0000","last_up_change":"2024-09-14T03:45:53.777419+0000","last_in_change":"2024-09-14T03:33:18.467778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":60,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:31:39.074672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:33:57.860054+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:34:01.436541+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:34:50.939554+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"72194dcf-0a09-4942-8ff9-82510a5a8011","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":311,"up_from":324,"up_thru":1054,"down_at":312,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":4018695393},{"type":"v1","addr":"172.21.15.110:6803","nonce":4018695393}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":4018695393},{"type":"v1","addr":"172.21.15.110:6805","nonce":4018695393}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":4018695393},{"type":"v1","addr":"172.21.15.110:6809","nonce":4018695393}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":4018695393},{"type":"v1","addr":"172.21.15.110:6807","nonce":4018695393}]},"public_addr":"172.21.15.110:6803/4018695393","cluster_addr":"172.21.15.110:6805/4018695393","heartbeat_back_addr":"172.21.15.110:6809/4018695393","heartbeat_front_addr":"172.21.15.110:6807/4018695393","state":["exists","up"]},{"osd":1,"uuid":"91257255-3d84-4da6-b336-e878b7a021b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":344,"up_from":358,"up_thru":1054,"down_at":345,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":1059959418},{"type":"v1","addr":"172.21.15.110:6811","nonce":1059959418}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":1059959418},{"type":"v1","addr":"172.21.15.110:6813","nonce":1059959418}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":1059959418},{"type":"v1","addr":"172.21.15.110:6817","nonce":1059959418}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":1059959418},{"type":"v1","addr":"172.21.15.110:6815","nonce":1059959418}]},"public_addr":"172.21.15.110:6811/1059959418","cluster_addr":"172.21.15.110:6813/1059959418","heartbeat_back_addr":"172.21.15.110:6817/1059959418","heartbeat_front_addr":"172.21.15.110:6815/1059959418","state":["exists","up"]},{"osd":2,"uuid":"f3e36fd0-3d58-4fa5-9ec9-8ea0b971183d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":376,"up_from":389,"up_thru":1054,"down_at":377,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":2442181625},{"type":"v1","addr":"172.21.15.110:6819","nonce":2442181625}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":2442181625},{"type":"v1","addr":"172.21.15.110:6821","nonce":2442181625}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2442181625},{"type":"v1","addr":"172.21.15.110:6825","nonce":2442181625}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":2442181625},{"type":"v1","addr":"172.21.15.110:6823","nonce":2442181625}]},"public_addr":"172.21.15.110:6819/2442181625","cluster_addr":"172.21.15.110:6821/2442181625","heartbeat_back_addr":"172.21.15.110:6825/2442181625","heartbeat_front_addr":"172.21.15.110:6823/2442181625","state":["exists","up"]},{"osd":3,"uuid":"38519016-56eb-4523-b648-fe57a8bc35b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":409,"up_from":420,"up_thru":1054,"down_at":410,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":570098450},{"type":"v1","addr":"172.21.15.110:6827","nonce":570098450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":570098450},{"type":"v1","addr":"172.21.15.110:6829","nonce":570098450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":570098450},{"type":"v1","addr":"172.21.15.110:6833","nonce":570098450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":570098450},{"type":"v1","addr":"172.21.15.110:6831","nonce":570098450}]},"public_addr":"172.21.15.110:6827/570098450","cluster_addr":"172.21.15.110:6829/570098450","heartbeat_back_addr":"172.21.15.110:6833/570098450","heartbeat_front_addr":"172.21.15.110:6831/570098450","state":["exists","up"]},{"osd":4,"uuid":"76641e65-6c1a-4adc-9c67-e6c89dc248e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":440,"up_from":454,"up_thru":1054,"down_at":441,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":2754374122},{"type":"v1","addr":"172.21.15.195:6801","nonce":2754374122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":2754374122},{"type":"v1","addr":"172.21.15.195:6803","nonce":2754374122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":2754374122},{"type":"v1","addr":"172.21.15.195:6807","nonce":2754374122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":2754374122},{"type":"v1","addr":"172.21.15.195:6805","nonce":2754374122}]},"public_addr":"172.21.15.195:6801/2754374122","cluster_addr":"172.21.15.195:6803/2754374122","heartbeat_back_addr":"172.21.15.195:6807/2754374122","heartbeat_front_addr":"172.21.15.195:6805/2754374122","state":["exists","up"]},{"osd":5,"uuid":"d7c7ab2e-8a49-4fcc-ba9d-5255a21c5fd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":37,"last_clean_end":474,"up_from":486,"up_thru":1054,"down_at":475,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3630816993},{"type":"v1","addr":"172.21.15.195:6809","nonce":3630816993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3630816993},{"type":"v1","addr":"172.21.15.195:6811","nonce":3630816993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3630816993},{"type":"v1","addr":"172.21.15.195:6815","nonce":3630816993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3630816993},{"type":"v1","addr":"172.21.15.195:6813","nonce":3630816993}]},"public_addr":"172.21.15.195:6809/3630816993","cluster_addr":"172.21.15.195:6811/3630816993","heartbeat_back_addr":"172.21.15.195:6815/3630816993","heartbeat_front_addr":"172.21.15.195:6813/3630816993","state":["exists","up"]},{"osd":6,"uuid":"cc7c808d-655e-4183-b18f-f6b57a6a5797","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":43,"last_clean_end":505,"up_from":518,"up_thru":1054,"down_at":506,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":977836419},{"type":"v1","addr":"172.21.15.195:6817","nonce":977836419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":977836419},{"type":"v1","addr":"172.21.15.195:6819","nonce":977836419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":977836419},{"type":"v1","addr":"172.21.15.195:6823","nonce":977836419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":977836419},{"type":"v1","addr":"172.21.15.195:6821","nonce":977836419}]},"public_addr":"172.21.15.195:6817/977836419","cluster_addr":"172.21.15.195:6819/977836419","heartbeat_back_addr":"172.21.15.195:6823/977836419","heartbeat_front_addr":"172.21.15.195:6821/977836419","state":["exists","up"]},{"osd":7,"uuid":"b7b35908-a28c-4497-a688-ac61826e57ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":49,"last_clean_end":539,"up_from":552,"up_thru":1054,"down_at":540,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":72177926},{"type":"v1","addr":"172.21.15.195:6825","nonce":72177926}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":72177926},{"type":"v1","addr":"172.21.15.195:6827","nonce":72177926}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":72177926},{"type":"v1","addr":"172.21.15.195:6831","nonce":72177926}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":72177926},{"type":"v1","addr":"172.21.15.195:6829","nonce":72177926}]},"public_addr":"172.21.15.195:6825/72177926","cluster_addr":"172.21.15.195:6827/72177926","heartbeat_back_addr":"172.21.15.195:6831/72177926","heartbeat_front_addr":"172.21.15.195:6829/72177926","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:30:50.215378+0000","dead_epoch":311},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:13.521829+0000","dead_epoch":343},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:31:36.212128+0000","dead_epoch":376},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:00.167368+0000","dead_epoch":409},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:21.694269+0000","dead_epoch":440},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:32:45.194575+0000","dead_epoch":474},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:09.117265+0000","dead_epoch":505},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:33:33.498914+0000","dead_epoch":538}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1b","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":2},{"from":6,"to":1}]},{"pgid":"3.20","mappings":[{"from":0,"to":2}]},{"pgid":"3.23","mappings":[{"from":0,"to":2}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.27","mappings":[{"from":4,"to":7}]},{"pgid":"3.30","mappings":[{"from":0,"to":1},{"from":5,"to":2}]},{"pgid":"3.3a","mappings":[{"from":5,"to":7}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/736862685":"2024-09-15T03:40:41.692273+0000","172.21.15.195:6833/1180864653":"2024-09-15T03:46:16.618713+0000","172.21.15.110:0/2397423297":"2024-09-15T03:40:41.692273+0000","172.21.15.110:0/1810777786":"2024-09-15T03:40:41.692273+0000","172.21.15.110:6800/2030214503":"2024-09-15T03:40:41.692273+0000","172.21.15.110:0/2220961304":"2024-09-15T03:40:41.692273+0000","172.21.15.110:0/2384140794":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/3893271533":"2024-09-15T03:38:59.210484+0000","172.21.15.110:6834/2015946497":"2024-09-15T03:45:58.999716+0000","172.21.15.110:0/974848163":"2024-09-15T03:40:41.692273+0000","172.21.15.110:6801/1862102623":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/2803573586":"2024-09-15T03:40:41.692273+0000","172.21.15.110:0/842858805":"2024-09-15T03:38:59.210484+0000","172.21.15.110:6801/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:0/2021495434":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2963489706":"2024-09-15T03:38:59.210484+0000","172.21.15.195:0/3089178541":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/3520643553":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/1862102623":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/3888509802":"2024-09-15T03:38:59.210484+0000","172.21.15.110:6800/1440218930":"2024-09-15T03:38:59.210484+0000","172.21.15.110:0/3627034844":"2024-09-15T03:29:10.827353+0000","172.21.15.110:0/2856593023":"2024-09-15T03:34:49.809403+0000","172.21.15.110:0/2779556305":"2024-09-15T03:38:59.210484+0000","172.21.15.110:0/2604403853":"2024-09-15T03:29:32.777703+0000","172.21.15.195:6832/1180864653":"2024-09-15T03:46:16.618713+0000","172.21.15.110:0/444976420":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6800/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.195:0/1517485107":"2024-09-15T03:40:01.254040+0000","172.21.15.195:0/1330315713":"2024-09-15T03:40:01.254040+0000","172.21.15.195:0/2196232450":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/3881395783":"2024-09-15T03:40:41.692273+0000","172.21.15.110:6801/595458891":"2024-09-15T03:29:10.827353+0000","172.21.15.195:0/1384830929":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/4093221761":"2024-09-15T03:34:49.809403+0000","172.21.15.110:6801/1440218930":"2024-09-15T03:38:59.210484+0000","172.21.15.110:6800/1267517953":"2024-09-15T03:29:32.777703+0000","172.21.15.110:6801/2030214503":"2024-09-15T03:40:41.692273+0000","172.21.15.195:0/2908653763":"2024-09-15T03:40:01.254040+0000","172.21.15.195:0/145968583":"2024-09-15T03:40:01.254040+0000","172.21.15.110:6835/2015946497":"2024-09-15T03:45:58.999716+0000","172.21.15.195:6834/622119125":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/1769339162":"2024-09-15T03:34:49.809403+0000","172.21.15.195:6835/622119125":"2024-09-15T03:40:01.254040+0000","172.21.15.110:0/2214025689":"2024-09-15T03:34:49.809403+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":60,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:56:00.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:55:59 smithi195 ceph-mon[82388]: pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:00.708 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:56:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:56:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:00.736 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs fail cephfs 2024-09-14T03:56:00.987 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:01.107 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:00 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/3952508090' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T03:56:01.108 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:00 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/3952508090' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T03:56:01.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:00 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/3952508090' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T03:56:01.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:01 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:56:01] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-14T03:56:02.102 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:01 smithi110 ceph-mon[86423]: pgmap v1267: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:02.102 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:01 smithi110 ceph-mon[88262]: pgmap v1267: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:02.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:01 smithi195 ceph-mon[82388]: pgmap v1267: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:02.823 INFO:teuthology.orchestra.run.smithi110.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-09-14T03:56:02.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[86419]: 2024-09-14T03:56:02.819+0000 7fa26c043640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:56:02.889 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[111676]: -4806> 2024-09-14T03:46:21.766+0000 7fca01337640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-14T03:56:02.889 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[111676]: ignoring --setuser ceph since I am not root 2024-09-14T03:56:02.890 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[111676]: ignoring --setgroup ceph since I am not root 2024-09-14T03:56:02.890 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[111676]: starting mds.a at 2024-09-14T03:56:03.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[88262]: pgmap v1268: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:03.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2451963625' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T03:56:03.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[88262]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:56:03.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[88262]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:56:03.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/2451963625' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-14T03:56:03.145 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[88262]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T03:56:03.145 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[88262]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:56:03.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[86423]: pgmap v1268: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:03.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2451963625' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T03:56:03.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[86423]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:56:03.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[86423]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:56:03.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/2451963625' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-14T03:56:03.146 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[86423]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T03:56:03.146 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:02 smithi110 ceph-mon[86423]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:56:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:02 smithi195 ceph-mon[82388]: pgmap v1268: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:02 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2451963625' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T03:56:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:02 smithi195 ceph-mon[82388]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:56:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:02 smithi195 ceph-mon[82388]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:56:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:02 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/2451963625' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-14T03:56:03.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:02 smithi195 ceph-mon[82388]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T03:56:03.210 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:02 smithi195 ceph-mon[82388]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:56:03.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph fs rm cephfs --yes-i-really-mean-it 2024-09-14T03:56:03.665 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:04.037 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:03 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:04.038 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:03 smithi110 ceph-mon[88262]: mds.? [v2:172.21.15.110:6834/808943509,v1:172.21.15.110:6835/808943509] up:boot 2024-09-14T03:56:04.038 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:03 smithi110 ceph-mon[88262]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-14T03:56:04.038 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:03 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:04.038 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:03 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:04.038 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:03 smithi110 ceph-mon[86423]: mds.? [v2:172.21.15.110:6834/808943509,v1:172.21.15.110:6835/808943509] up:boot 2024-09-14T03:56:04.038 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:03 smithi110 ceph-mon[86423]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-14T03:56:04.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:03 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:04.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:03 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:04.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:03 smithi195 ceph-mon[82388]: mds.? [v2:172.21.15.110:6834/808943509,v1:172.21.15.110:6835/808943509] up:boot 2024-09-14T03:56:04.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:03 smithi195 ceph-mon[82388]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-14T03:56:04.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:03 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:05.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:04 smithi110 ceph-mon[88262]: pgmap v1270: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:05.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:04 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:05.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:04 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:05.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:04 smithi110 ceph-mon[88262]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:56:05.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:04 smithi110 ceph-mon[86423]: pgmap v1270: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:05.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:04 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:05.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:04 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:05.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:04 smithi110 ceph-mon[86423]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:56:05.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:04 smithi195 ceph-mon[82388]: pgmap v1270: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:04 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:04 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:05.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:04 smithi195 ceph-mon[82388]: from='mgr.34107 ' entity='mgr.y' 2024-09-14T03:56:06.260 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:05 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/1698064653' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:56:06.260 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:05 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:56:06.260 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:05 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/1698064653' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:56:06.261 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:05 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:56:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:05 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/1698064653' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:56:06.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:05 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:56:06.571 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-09-14T03:56:06.817 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[88262]: pgmap v1271: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[88262]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:56:07.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[88262]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:56:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[88262]: Cluster is now healthy 2024-09-14T03:56:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[88262]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:56:07.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[88262]: fsmap 2 up:standby 2024-09-14T03:56:07.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[86423]: pgmap v1271: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:07.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[86423]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:56:07.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[86423]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:56:07.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[86423]: Cluster is now healthy 2024-09-14T03:56:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[86423]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:56:07.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:06 smithi110 ceph-mon[86423]: fsmap 2 up:standby 2024-09-14T03:56:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:06 smithi195 ceph-mon[82388]: pgmap v1271: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:06 smithi195 ceph-mon[82388]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:56:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:06 smithi195 ceph-mon[82388]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:56:07.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:06 smithi195 ceph-mon[82388]: Cluster is now healthy 2024-09-14T03:56:07.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:06 smithi195 ceph-mon[82388]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:56:07.460 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:06 smithi195 ceph-mon[82388]: fsmap 2 up:standby 2024-09-14T03:56:08.972 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'cephfs_metadata' removed 2024-09-14T03:56:09.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:08 smithi110 ceph-mon[88262]: pgmap v1272: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:09.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:08 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4020802152' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:09.353 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:08 smithi110 ceph-mon[86423]: pgmap v1272: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:09.353 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:08 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4020802152' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:09.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:08 smithi195 ceph-mon[82388]: pgmap v1272: 105 pgs: 105 active+clean; 583 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:09.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:08 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4020802152' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:09.649 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 5517b266-7249-11ef-bceb-c7b262605968 -- ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-09-14T03:56:09.897 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:10.233 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:09 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/4020802152' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:10.233 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:09 smithi110 ceph-mon[88262]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T03:56:10.233 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:09 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/4020802152' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:10.233 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:09 smithi110 ceph-mon[86423]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T03:56:10.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:09 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/4020802152' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:10.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:09 smithi195 ceph-mon[82388]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T03:56:10.459 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:56:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:56:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:11.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:10 smithi110 ceph-mon[88262]: pgmap v1274: 73 pgs: 73 active+clean; 577 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:11.329 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:10 smithi110 ceph-mon[86423]: pgmap v1274: 73 pgs: 73 active+clean; 577 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:11.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:10 smithi195 ceph-mon[82388]: pgmap v1274: 73 pgs: 73 active+clean; 577 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:11.696 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:11 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: ::ffff:172.21.15.195 - - [14/Sep/2024:03:56:11] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-14T03:56:11.981 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'cephfs_data' removed 2024-09-14T03:56:12.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:11 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/219824545' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:12.301 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:11 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/219824545' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:12.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:11 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/219824545' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:12.626 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:12.653 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:12.689 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-14T03:56:12.689 DEBUG:teuthology.orchestra.run.smithi110:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-14T03:56:12.708 DEBUG:teuthology.orchestra.run.smithi195:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-14T03:56:12.743 INFO:tasks.cephadm:Stopping all daemons... 2024-09-14T03:56:12.744 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-14T03:56:12.744 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a 2024-09-14T03:56:12.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:12 smithi110 systemd[1]: Stopping Ceph mon.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:13.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:12 smithi110 ceph-mon[88262]: pgmap v1275: 73 pgs: 73 active+clean; 577 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:13.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:12 smithi110 ceph-mon[88262]: from='client.? 172.21.15.110:0/219824545' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:13.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:12 smithi110 ceph-mon[88262]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T03:56:13.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:12 smithi110 ceph-mon[88262]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:13.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:12 smithi110 ceph-mon[86423]: pgmap v1275: 73 pgs: 73 active+clean; 577 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:13.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:12 smithi110 ceph-mon[86423]: from='client.? 172.21.15.110:0/219824545' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:13.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:12 smithi110 ceph-mon[86423]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T03:56:13.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:12 smithi110 ceph-mon[86423]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:13.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:13 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[86419]: 2024-09-14T03:56:13.063+0000 7fa27184e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:13.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:13 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a[86419]: 2024-09-14T03:56:13.063+0000 7fa27184e640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-14T03:56:13.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:13 smithi110 podman[119748]: 2024-09-14 03:56:13.094341629 +0000 UTC m=+0.151154108 container died c325cb2844f41afa7669ff7ca8517c47c716a8479eb3b5110e63a1b185d9e405 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-14T03:56:13.458 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:12 smithi195 ceph-mon[82388]: pgmap v1275: 73 pgs: 73 active+clean; 577 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:56:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:12 smithi195 ceph-mon[82388]: from='client.? 172.21.15.110:0/219824545' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:12 smithi195 ceph-mon[82388]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T03:56:13.459 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:12 smithi195 ceph-mon[82388]: from='mgr.34107 172.21.15.110:0/883906958' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:13.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:13 smithi110 podman[119748]: 2024-09-14 03:56:13.464044754 +0000 UTC m=+0.520857265 container cleanup c325cb2844f41afa7669ff7ca8517c47c716a8479eb3b5110e63a1b185d9e405 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T03:56:13.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 14 03:56:13 smithi110 bash[119748]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:56:13.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.a.service' 2024-09-14T03:56:13.897 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:13.898 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-14T03:56:13.898 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-14T03:56:13.898 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.c 2024-09-14T03:56:14.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:13 smithi110 systemd[1]: Stopping Ceph mon.c for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:14.570 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:56:14] ENGINE Bus STOPPING 2024-09-14T03:56:14.570 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c[88258]: 2024-09-14T03:56:14.226+0000 7fd20c12e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:14.571 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c[88258]: 2024-09-14T03:56:14.226+0000 7fd20c12e640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-14T03:56:14.571 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:14 smithi110 podman[119871]: 2024-09-14 03:56:14.420941399 +0000 UTC m=+0.305357912 container died a2cf589f8f7e8dd48b8d0f40d7b583eff63a0f1c8598f95dd1e7a001d8bcf7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T03:56:14.828 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:56:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:56:14.828 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:56:14] ENGINE Bus STOPPED 2024-09-14T03:56:14.828 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:56:14] ENGINE Bus STARTING 2024-09-14T03:56:14.828 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:56:14] ENGINE Serving on http://:::9283 2024-09-14T03:56:14.829 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:14 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: [14/Sep/2024:03:56:14] ENGINE Bus STARTED 2024-09-14T03:56:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:14 smithi110 podman[119871]: 2024-09-14 03:56:14.827464067 +0000 UTC m=+0.711880584 container cleanup a2cf589f8f7e8dd48b8d0f40d7b583eff63a0f1c8598f95dd1e7a001d8bcf7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T03:56:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:14 smithi110 bash[119871]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c 2024-09-14T03:56:15.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 14 03:56:15 smithi110 podman[119918]: 2024-09-14 03:56:15.070712453 +0000 UTC m=+0.650337739 container remove a2cf589f8f7e8dd48b8d0f40d7b583eff63a0f1c8598f95dd1e7a001d8bcf7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-c, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-14T03:56:15.234 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.c.service' 2024-09-14T03:56:15.271 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:15.271 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-14T03:56:15.272 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-14T03:56:15.272 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.b 2024-09-14T03:56:15.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:15 smithi195 systemd[1]: Stopping Ceph mon.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:15.958 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b[82384]: 2024-09-14T03:56:15.573+0000 7f373fb2c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:15.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b[82384]: 2024-09-14T03:56:15.573+0000 7f373fb2c640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-14T03:56:15.959 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:15 smithi195 podman[113240]: 2024-09-14 03:56:15.685673899 +0000 UTC m=+0.223661769 container died 4992d17f309cebba5053cbbdbbf6c09d3a9c105d677e5fa9460e384b56dde1db (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231) 2024-09-14T03:56:16.208 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:16 smithi195 podman[113240]: 2024-09-14 03:56:16.156647176 +0000 UTC m=+0.694635074 container cleanup 4992d17f309cebba5053cbbdbbf6c09d3a9c105d677e5fa9460e384b56dde1db (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231) 2024-09-14T03:56:16.209 INFO:journalctl@ceph.mon.b.smithi195.stdout:Sep 14 03:56:16 smithi195 bash[113240]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mon-b 2024-09-14T03:56:16.546 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mon.b.service' 2024-09-14T03:56:16.594 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:16.594 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-14T03:56:16.594 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-14T03:56:16.595 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y 2024-09-14T03:56:16.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:16 smithi110 systemd[1]: Stopping Ceph mgr.y for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:18.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:18 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y[78674]: 2024-09-14T03:56:18.185+0000 7f946cb26640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:18.446 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:18 smithi110 podman[120429]: 2024-09-14 03:56:18.217054544 +0000 UTC m=+0.483793085 container died f1f945bddfa635ec5c24e58255d0568ab8cee6cfeb5b56de31fbef73e280ccb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:56:19.055 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:18 smithi110 podman[120429]: 2024-09-14 03:56:18.870085866 +0000 UTC m=+1.136824391 container cleanup f1f945bddfa635ec5c24e58255d0568ab8cee6cfeb5b56de31fbef73e280ccb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, ceph=True, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-14T03:56:19.056 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:18 smithi110 bash[120429]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y 2024-09-14T03:56:19.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-14T03:56:19.314 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:19 smithi110 podman[120443]: 2024-09-14 03:56:19.054938422 +0000 UTC m=+0.831724550 container remove f1f945bddfa635ec5c24e58255d0568ab8cee6cfeb5b56de31fbef73e280ccb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-14T03:56:19.315 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:19 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service: Deactivated successfully. 2024-09-14T03:56:19.315 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:19 smithi110 systemd[1]: Stopped Ceph mgr.y for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:56:19.315 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 14 03:56:19 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.y.service: Consumed 28.783s CPU time. 2024-09-14T03:56:19.330 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:19.330 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-14T03:56:19.331 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-14T03:56:19.331 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x 2024-09-14T03:56:19.668 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:56:19 smithi195 systemd[1]: Stopping Ceph mgr.x for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:19.921 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:56:19 smithi195 podman[113364]: 2024-09-14 03:56:19.667221636 +0000 UTC m=+0.144906143 container died fbeb4897d8684a5b6de7209b7db4f23e04da1c84f14ec694d040cee0b099531a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-14T03:56:20.208 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:56:20 smithi195 podman[113364]: 2024-09-14 03:56:20.002719998 +0000 UTC m=+0.480404515 container cleanup fbeb4897d8684a5b6de7209b7db4f23e04da1c84f14ec694d040cee0b099531a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:56:20.209 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Sep 14 03:56:20 smithi195 bash[113364]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:56:20.428 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-14T03:56:20.464 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:20.464 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-14T03:56:20.465 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-14T03:56:20.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.0 2024-09-14T03:56:20.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:56:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:56:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"5517b266-7249-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:20.792 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:20 smithi110 systemd[1]: Stopping Ceph osd.0 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:21.196 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[93555]: 2024-09-14T03:56:20.790+0000 7f09b825b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:21.196 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[93555]: 2024-09-14T03:56:20.790+0000 7f09b825b640 -1 osd.0 1062 *** Got signal Terminated *** 2024-09-14T03:56:21.196 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:20 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0[93555]: 2024-09-14T03:56:20.790+0000 7f09b825b640 -1 osd.0 1062 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:56:26.619 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:26 smithi110 podman[120550]: 2024-09-14 03:56:26.363089818 +0000 UTC m=+5.702759180 container died 9626562f2155f97ad95c41a59b987d4502079930d0e84c7d276b20426e4a8fbf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:26.901 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:26 smithi110 podman[120550]: 2024-09-14 03:56:26.690820031 +0000 UTC m=+6.030489381 container cleanup 9626562f2155f97ad95c41a59b987d4502079930d0e84c7d276b20426e4a8fbf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:26.902 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:26 smithi110 bash[120550]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0 2024-09-14T03:56:27.196 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:26 smithi110 podman[120566]: 2024-09-14 03:56:26.900657979 +0000 UTC m=+0.534576344 container remove 9626562f2155f97ad95c41a59b987d4502079930d0e84c7d276b20426e4a8fbf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:56:27.605 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:27 smithi110 podman[120656]: 2024-09-14 03:56:27.301528315 +0000 UTC m=+0.112484367 container create f925a45fd13b0894d9b2f2f1c95f304bfbde6620d2d026f31b07da2d87c1c677 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231) 2024-09-14T03:56:27.605 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:27 smithi110 podman[120656]: 2024-09-14 03:56:27.204927018 +0000 UTC m=+0.015883088 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:27.605 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:27 smithi110 podman[120656]: 2024-09-14 03:56:27.512674777 +0000 UTC m=+0.323630829 container init f925a45fd13b0894d9b2f2f1c95f304bfbde6620d2d026f31b07da2d87c1c677 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:27.606 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:27 smithi110 podman[120656]: 2024-09-14 03:56:27.515853439 +0000 UTC m=+0.326809495 container start f925a45fd13b0894d9b2f2f1c95f304bfbde6620d2d026f31b07da2d87c1c677 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731) 2024-09-14T03:56:27.606 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:27 smithi110 podman[120656]: 2024-09-14 03:56:27.604298946 +0000 UTC m=+0.415255003 container attach f925a45fd13b0894d9b2f2f1c95f304bfbde6620d2d026f31b07da2d87c1c677 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.build-date=20240731) 2024-09-14T03:56:27.946 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:27 smithi110 podman[120656]: 2024-09-14 03:56:27.699046763 +0000 UTC m=+0.510002816 container died f925a45fd13b0894d9b2f2f1c95f304bfbde6620d2d026f31b07da2d87c1c677 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231) 2024-09-14T03:56:28.244 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 14 03:56:28 smithi110 podman[120679]: 2024-09-14 03:56:28.241449043 +0000 UTC m=+0.536976605 container remove f925a45fd13b0894d9b2f2f1c95f304bfbde6620d2d026f31b07da2d87c1c677 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-14T03:56:28.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.0.service' 2024-09-14T03:56:28.303 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:28.304 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-14T03:56:28.304 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-14T03:56:28.304 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.1 2024-09-14T03:56:28.540 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:28 smithi110 systemd[1]: Stopping Ceph osd.1 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:28.946 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[97937]: 2024-09-14T03:56:28.744+0000 7f24aa314640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:28.946 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[97937]: 2024-09-14T03:56:28.744+0000 7f24aa314640 -1 osd.1 1062 *** Got signal Terminated *** 2024-09-14T03:56:28.946 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:28 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1[97937]: 2024-09-14T03:56:28.744+0000 7f24aa314640 -1 osd.1 1062 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:56:34.446 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:34 smithi110 podman[120764]: 2024-09-14 03:56:34.16831319 +0000 UTC m=+5.523021318 container died 6ed2d900d1e261d3b988047e95463380c2bb2e5710bc3c1cadf65256b7ff3fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:35.041 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:34 smithi110 podman[120764]: 2024-09-14 03:56:34.787226886 +0000 UTC m=+6.141935012 container cleanup 6ed2d900d1e261d3b988047e95463380c2bb2e5710bc3c1cadf65256b7ff3fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.schema-version=1.0) 2024-09-14T03:56:35.041 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:34 smithi110 bash[120764]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1 2024-09-14T03:56:35.446 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:35 smithi110 podman[120790]: 2024-09-14 03:56:35.039122344 +0000 UTC m=+0.864486100 container remove 6ed2d900d1e261d3b988047e95463380c2bb2e5710bc3c1cadf65256b7ff3fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:35.735 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:35 smithi110 podman[120881]: 2024-09-14 03:56:35.45649606 +0000 UTC m=+0.112966184 container create b5fcb0d9e23b4c350f65a239e6a3ce267737ad62d7d11967d0039d8bf1937a31 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T03:56:35.735 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:35 smithi110 podman[120881]: 2024-09-14 03:56:35.359315288 +0000 UTC m=+0.015785413 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:35.735 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:35 smithi110 podman[120881]: 2024-09-14 03:56:35.659019996 +0000 UTC m=+0.315490124 container init b5fcb0d9e23b4c350f65a239e6a3ce267737ad62d7d11967d0039d8bf1937a31 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T03:56:35.735 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:35 smithi110 podman[120881]: 2024-09-14 03:56:35.66220825 +0000 UTC m=+0.318678384 container start b5fcb0d9e23b4c350f65a239e6a3ce267737ad62d7d11967d0039d8bf1937a31 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=main-a278231, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T03:56:36.196 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:35 smithi110 podman[120881]: 2024-09-14 03:56:35.733922508 +0000 UTC m=+0.390392651 container attach b5fcb0d9e23b4c350f65a239e6a3ce267737ad62d7d11967d0039d8bf1937a31 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:56:36.196 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 14 03:56:35 smithi110 podman[120881]: 2024-09-14 03:56:35.851956439 +0000 UTC m=+0.508426565 container died b5fcb0d9e23b4c350f65a239e6a3ce267737ad62d7d11967d0039d8bf1937a31 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-14T03:56:36.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.1.service' 2024-09-14T03:56:36.644 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:36.644 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-14T03:56:36.644 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-14T03:56:36.645 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.2 2024-09-14T03:56:36.946 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:36 smithi110 systemd[1]: Stopping Ceph osd.2 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:37.446 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:36 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:56:36.972+0000 7fba51386640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:37.446 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:36 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:56:36.972+0000 7fba51386640 -1 osd.2 1062 *** Got signal Terminated *** 2024-09-14T03:56:37.446 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:36 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2[102310]: 2024-09-14T03:56:36.972+0000 7fba51386640 -1 osd.2 1062 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:56:42.843 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:42 smithi110 podman[120985]: 2024-09-14 03:56:42.547376127 +0000 UTC m=+5.682492355 container died 0cdc58c38711a34029d3f27d974cdbe4f2e83dca3c80164b2e484a24d7e68e05 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:43.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:43 smithi110 podman[120985]: 2024-09-14 03:56:43.041298389 +0000 UTC m=+6.176414616 container cleanup 0cdc58c38711a34029d3f27d974cdbe4f2e83dca3c80164b2e484a24d7e68e05 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-14T03:56:43.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:43 smithi110 bash[120985]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2 2024-09-14T03:56:43.628 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:43 smithi110 podman[121000]: 2024-09-14 03:56:43.234476874 +0000 UTC m=+0.686350855 container remove 0cdc58c38711a34029d3f27d974cdbe4f2e83dca3c80164b2e484a24d7e68e05 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T03:56:43.897 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:43 smithi110 podman[121090]: 2024-09-14 03:56:43.627363476 +0000 UTC m=+0.093360180 container create 63389a095cd126a4f0afd8df76003b5b4af05007df8d57e0053c0bd24a132b23 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T03:56:43.897 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:43 smithi110 podman[121090]: 2024-09-14 03:56:43.549512331 +0000 UTC m=+0.015508994 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:43.898 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:43 smithi110 podman[121090]: 2024-09-14 03:56:43.821430059 +0000 UTC m=+0.287426726 container init 63389a095cd126a4f0afd8df76003b5b4af05007df8d57e0053c0bd24a132b23 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T03:56:43.898 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:43 smithi110 podman[121090]: 2024-09-14 03:56:43.824615483 +0000 UTC m=+0.290612152 container start 63389a095cd126a4f0afd8df76003b5b4af05007df8d57e0053c0bd24a132b23 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True) 2024-09-14T03:56:43.898 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:43 smithi110 podman[121090]: 2024-09-14 03:56:43.896386455 +0000 UTC m=+0.362383136 container attach 63389a095cd126a4f0afd8df76003b5b4af05007df8d57e0053c0bd24a132b23 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True) 2024-09-14T03:56:44.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:44 smithi110 podman[121090]: 2024-09-14 03:56:44.011706451 +0000 UTC m=+0.477703119 container died 63389a095cd126a4f0afd8df76003b5b4af05007df8d57e0053c0bd24a132b23 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:44.553 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 14 03:56:44 smithi110 podman[121113]: 2024-09-14 03:56:44.533171736 +0000 UTC m=+0.515573965 container remove 63389a095cd126a4f0afd8df76003b5b4af05007df8d57e0053c0bd24a132b23 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-14T03:56:44.558 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.2.service' 2024-09-14T03:56:44.596 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:44.596 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-14T03:56:44.596 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-14T03:56:44.596 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.3 2024-09-14T03:56:44.912 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:44 smithi110 systemd[1]: Stopping Ceph osd.3 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:45.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:44 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[106966]: 2024-09-14T03:56:44.910+0000 7fdb9aa69640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:45.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:44 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[106966]: 2024-09-14T03:56:44.910+0000 7fdb9aa69640 -1 osd.3 1062 *** Got signal Terminated *** 2024-09-14T03:56:45.196 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:44 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3[106966]: 2024-09-14T03:56:44.910+0000 7fdb9aa69640 -1 osd.3 1062 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:56:47.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:47.637+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:47.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:47 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:47.400+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:48.675 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:48.235+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:48.676 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:48.410+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:48.929 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:48 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:48.673+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:49.627 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:49.383+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:49.628 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:49.245+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:49.883 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:49.626+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:49.884 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:49.664+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:49.884 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:56:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=sqlstore.transactions t=2024-09-14T03:56:49.737945604Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:56:49.884 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:56:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=cleanup t=2024-09-14T03:56:49.744121141Z level=info msg="Completed cleanup jobs" duration=6.442023ms 2024-09-14T03:56:49.885 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:56:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=grafana.update.checker t=2024-09-14T03:56:49.883017578Z level=info msg="Update check succeeded" duration=68.298944ms 2024-09-14T03:56:50.209 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:56:49 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=plugins.update.checker t=2024-09-14T03:56:49.885833244Z level=info msg="Update check succeeded" duration=69.104126ms 2024-09-14T03:56:50.651 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:50.346+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:50.652 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:50.225+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:50.662 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:50 smithi110 podman[121193]: 2024-09-14 03:56:50.399662035 +0000 UTC m=+5.584131735 container died 2283ebcb5f1004d507c60ac6e1c762b6c8ecf9507da384880ecd68b6a58fb17e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-a278231, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:56:50.946 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:50 smithi110 podman[121193]: 2024-09-14 03:56:50.719323615 +0000 UTC m=+5.903793365 container cleanup 2283ebcb5f1004d507c60ac6e1c762b6c8ecf9507da384880ecd68b6a58fb17e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-14T03:56:50.946 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:50 smithi110 bash[121193]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3 2024-09-14T03:56:50.946 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:50 smithi110 podman[121209]: 2024-09-14 03:56:50.903953639 +0000 UTC m=+0.505096570 container remove 2283ebcb5f1004d507c60ac6e1c762b6c8ecf9507da384880ecd68b6a58fb17e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T03:56:50.958 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:50.650+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:50.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:50 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:50.701+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:51.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:51 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:51.176+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:51.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:51 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:51.385+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:51.616 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:51 smithi110 podman[121296]: 2024-09-14 03:56:51.312976619 +0000 UTC m=+0.099414206 container create cc5f5b20f543db8cea1802f51421401e4b24478cc081df6b029bac7aef7902b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-14T03:56:51.617 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:51 smithi110 podman[121296]: 2024-09-14 03:56:51.228998584 +0000 UTC m=+0.015436174 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:51.617 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:51 smithi110 podman[121296]: 2024-09-14 03:56:51.540309074 +0000 UTC m=+0.326746653 container init cc5f5b20f543db8cea1802f51421401e4b24478cc081df6b029bac7aef7902b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-14T03:56:51.617 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:51 smithi110 podman[121296]: 2024-09-14 03:56:51.543629423 +0000 UTC m=+0.330067002 container start cc5f5b20f543db8cea1802f51421401e4b24478cc081df6b029bac7aef7902b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2) 2024-09-14T03:56:51.948 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:51 smithi110 podman[121296]: 2024-09-14 03:56:51.615308838 +0000 UTC m=+0.401746417 container attach cc5f5b20f543db8cea1802f51421401e4b24478cc081df6b029bac7aef7902b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:56:51.949 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:51 smithi110 podman[121296]: 2024-09-14 03:56:51.737824714 +0000 UTC m=+0.524262307 container died cc5f5b20f543db8cea1802f51421401e4b24478cc081df6b029bac7aef7902b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T03:56:51.958 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:51 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:51.683+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:51.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:51 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:51.744+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:52.287 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 14 03:56:52 smithi110 podman[121320]: 2024-09-14 03:56:52.269027746 +0000 UTC m=+0.525454307 container remove cc5f5b20f543db8cea1802f51421401e4b24478cc081df6b029bac7aef7902b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-14T03:56:52.291 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.3.service' 2024-09-14T03:56:52.329 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:56:52.329 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-14T03:56:52.329 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-14T03:56:52.329 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.4 2024-09-14T03:56:52.459 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:52 smithi195 systemd[1]: Stopping Ceph osd.4 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:56:52.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:52.200+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:52.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:52.418+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:52.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:52.788+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:52.959 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:52.635+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:52.959 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:52.637+0000 7f2bdbc55640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:52.959 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:52.637+0000 7f2bdbc55640 -1 osd.4 1062 *** Got signal Terminated *** 2024-09-14T03:56:52.960 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:52 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:52.637+0000 7f2bdbc55640 -1 osd.4 1062 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:56:53.456 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:53 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:53.170+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:53.709 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:53 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:53.684+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:53.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:53 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:53.454+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:54.205 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:53 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:53.790+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:54.458 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:54.204+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:54.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:54.438+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:54.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:54.438+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:56:55.191 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:54.728+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:55.191 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:54.761+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:55.191 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:54 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:54.761+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:56:55.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:55.189+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:55.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:55.396+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:55.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:55.396+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:56:56.193 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:55.726+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:56.193 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:55.720+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:56.193 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:55 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:55.720+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:56:56.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:56.191+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:56.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:56.417+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:56.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:56.417+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:56:56.958 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4[87084]: 2024-09-14T03:56:56.699+0000 7f2bd826a640 -1 osd.4 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.964332+0000 front 2024-09-14T03:56:20.964291+0000 (oldest deadline 2024-09-14T03:56:46.864176+0000) 2024-09-14T03:56:56.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:56.742+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:56.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:56 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:56.742+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:56:57.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:57 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:57.207+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:57.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:57 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:57.404+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:57.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:57 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:57.405+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:56:58.052 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:57 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:57.714+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:58.053 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:57 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:57.714+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:56:58.374 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:58 smithi195 podman[113501]: 2024-09-14 03:56:58.05167554 +0000 UTC m=+5.516948903 container died 6921b0a30047454dce0ad05a50f5fcc27b1f7841cab6b161f05557566cccd172 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-a278231, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS) 2024-09-14T03:56:58.374 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:58 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:58.238+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:58.374 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:58 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:58.238+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.447212+0000 front 2024-09-14T03:56:31.447146+0000 (oldest deadline 2024-09-14T03:56:57.347000+0000) 2024-09-14T03:56:58.690 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:58 smithi195 podman[113501]: 2024-09-14 03:56:58.472181427 +0000 UTC m=+5.937454792 container cleanup 6921b0a30047454dce0ad05a50f5fcc27b1f7841cab6b161f05557566cccd172 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-a278231, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-14T03:56:58.691 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:58 smithi195 bash[113501]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4 2024-09-14T03:56:58.691 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:58 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:58.427+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:58.691 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:58 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:58.427+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:56:58.958 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:58 smithi195 podman[113516]: 2024-09-14 03:56:58.690077396 +0000 UTC m=+0.640526008 container remove 6921b0a30047454dce0ad05a50f5fcc27b1f7841cab6b161f05557566cccd172 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T03:56:58.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:58 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:58.739+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:58.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:58 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:58.739+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:56:59.268 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:59 smithi195 podman[113608]: 2024-09-14 03:56:59.049737132 +0000 UTC m=+0.016040509 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:59.268 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:59 smithi195 podman[113608]: 2024-09-14 03:56:59.157649383 +0000 UTC m=+0.123952747 container create 6e85862720488835bcf31c5028a7ff4e5f30824ba641d7792a9481a6b39f9cd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-a278231) 2024-09-14T03:56:59.583 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:59.471+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:56:59.583 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:56:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:56:59.471+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:56:59.583 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:59 smithi195 podman[113608]: 2024-09-14 03:56:59.393552638 +0000 UTC m=+0.359856013 container init 6e85862720488835bcf31c5028a7ff4e5f30824ba641d7792a9481a6b39f9cd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-14T03:56:59.583 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:59 smithi195 podman[113608]: 2024-09-14 03:56:59.397116058 +0000 UTC m=+0.363419426 container start 6e85862720488835bcf31c5028a7ff4e5f30824ba641d7792a9481a6b39f9cd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:56:59.583 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:59 smithi195 podman[113608]: 2024-09-14 03:56:59.460127512 +0000 UTC m=+0.426430885 container attach 6e85862720488835bcf31c5028a7ff4e5f30824ba641d7792a9481a6b39f9cd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-14T03:56:59.584 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:59.286+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:56:59.584 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:56:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:56:59.286+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.447212+0000 front 2024-09-14T03:56:31.447146+0000 (oldest deadline 2024-09-14T03:56:57.347000+0000) 2024-09-14T03:56:59.889 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 14 03:56:59 smithi195 podman[113608]: 2024-09-14 03:56:59.582652169 +0000 UTC m=+0.548955535 container died 6e85862720488835bcf31c5028a7ff4e5f30824ba641d7792a9481a6b39f9cd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-09-14T03:56:59.890 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:59.739+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:56:59.890 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:56:59 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:56:59.739+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:00.200 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.4.service' 2024-09-14T03:57:00.242 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:57:00.243 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-14T03:57:00.243 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-14T03:57:00.243 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.5 2024-09-14T03:57:00.457 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:00.258+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:57:00.457 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:00.258+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.447212+0000 front 2024-09-14T03:56:31.447146+0000 (oldest deadline 2024-09-14T03:56:57.347000+0000) 2024-09-14T03:57:00.457 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:00 smithi195 systemd[1]: Stopping Ceph osd.5 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:00.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:00.456+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:00.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:00.456+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:00.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:00.591+0000 7f0e25589640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:57:00.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:00.591+0000 7f0e25589640 -1 osd.5 1062 *** Got signal Terminated *** 2024-09-14T03:57:00.710 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:00.591+0000 7f0e25589640 -1 osd.5 1062 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:57:01.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:00.717+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:01.209 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:00 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:00.717+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:01.495 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:01.216+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:57:01.495 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:01.216+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.447212+0000 front 2024-09-14T03:56:31.447146+0000 (oldest deadline 2024-09-14T03:56:57.347000+0000) 2024-09-14T03:57:01.958 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:01.711+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:01.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:01.711+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:01.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:01.494+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:01.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:01 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:01.494+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:02.531 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:02 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:02.247+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:57:02.531 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:02 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:02.247+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.447212+0000 front 2024-09-14T03:56:31.447146+0000 (oldest deadline 2024-09-14T03:56:57.347000+0000) 2024-09-14T03:57:02.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:02 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:02.688+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:02.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:02 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:02.688+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:02.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:02 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:02.530+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:02.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:02 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:02.530+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:03.512 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:03.212+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:57:03.513 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:03.212+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.447212+0000 front 2024-09-14T03:56:31.447146+0000 (oldest deadline 2024-09-14T03:56:57.347000+0000) 2024-09-14T03:57:03.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:03.737+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:03.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:03.737+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:03.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:03.511+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:03.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:03 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:03.511+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:04.511 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:04.210+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:57:04.511 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:04.210+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.447212+0000 front 2024-09-14T03:56:31.447146+0000 (oldest deadline 2024-09-14T03:56:57.347000+0000) 2024-09-14T03:57:04.511 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:04.210+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.448293+0000 front 2024-09-14T03:56:41.448212+0000 (oldest deadline 2024-09-14T03:57:03.748119+0000) 2024-09-14T03:57:04.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:04.746+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:04.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:04.746+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:04.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:04.509+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:04.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:04 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:04.509+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:05.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:05.203+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.346169+0000 front 2024-09-14T03:56:23.346084+0000 (oldest deadline 2024-09-14T03:56:47.445887+0000) 2024-09-14T03:57:05.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:05.203+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.447212+0000 front 2024-09-14T03:56:31.447146+0000 (oldest deadline 2024-09-14T03:56:57.347000+0000) 2024-09-14T03:57:05.459 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5[91798]: 2024-09-14T03:57:05.203+0000 7f0e21b9e640 -1 osd.5 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.448293+0000 front 2024-09-14T03:56:41.448212+0000 (oldest deadline 2024-09-14T03:57:03.748119+0000) 2024-09-14T03:57:05.726 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:05.469+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:05.726 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:05.469+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:06.118 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:05.724+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:06.118 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:05 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:05.724+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:06.386 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:06 smithi195 podman[113711]: 2024-09-14 03:57:06.117649656 +0000 UTC m=+5.637901611 container died 8fe7f08d58820a8a5a84f86c703180db19a805ebd5494f12a327d320833d525c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-a278231, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-14T03:57:06.703 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:06 smithi195 podman[113711]: 2024-09-14 03:57:06.492453168 +0000 UTC m=+6.012705121 container cleanup 8fe7f08d58820a8a5a84f86c703180db19a805ebd5494f12a327d320833d525c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-14T03:57:06.703 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:06 smithi195 bash[113711]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5 2024-09-14T03:57:06.703 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:06.492+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:06.704 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:06.492+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:06.704 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:06.492+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:06.958 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:06 smithi195 podman[113740]: 2024-09-14 03:57:06.743911177 +0000 UTC m=+0.616079668 container remove 8fe7f08d58820a8a5a84f86c703180db19a805ebd5494f12a327d320833d525c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:57:06.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:06.702+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:06.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:06 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:06.702+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:07.339 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:07 smithi195 podman[113830]: 2024-09-14 03:57:06.966247945 +0000 UTC m=+0.016996240 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:57:07.339 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:07 smithi195 podman[113830]: 2024-09-14 03:57:07.086426848 +0000 UTC m=+0.137175182 container create 8c554d97c87810b7b52f32da74fd03d878b96211a223817ba3e35005bf102134 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-14T03:57:07.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:07 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:07.450+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:07.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:07 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:07.450+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:07.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:07 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:07.450+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:07.709 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:07 smithi195 podman[113830]: 2024-09-14 03:57:07.33874472 +0000 UTC m=+0.389493017 container init 8c554d97c87810b7b52f32da74fd03d878b96211a223817ba3e35005bf102134 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:07.710 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:07 smithi195 podman[113830]: 2024-09-14 03:57:07.342228287 +0000 UTC m=+0.392976576 container start 8c554d97c87810b7b52f32da74fd03d878b96211a223817ba3e35005bf102134 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T03:57:07.710 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:07 smithi195 podman[113830]: 2024-09-14 03:57:07.447188112 +0000 UTC m=+0.497936418 container attach 8c554d97c87810b7b52f32da74fd03d878b96211a223817ba3e35005bf102134 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=main-a278231, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T03:57:07.710 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 14 03:57:07 smithi195 podman[113830]: 2024-09-14 03:57:07.5349264 +0000 UTC m=+0.585674696 container died 8c554d97c87810b7b52f32da74fd03d878b96211a223817ba3e35005bf102134 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T03:57:08.168 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:07 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:07.720+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:08.169 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:07 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:07.721+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:08.201 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.5.service' 2024-09-14T03:57:08.242 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:57:08.242 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-14T03:57:08.242 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-14T03:57:08.242 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.6 2024-09-14T03:57:08.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:08 smithi195 systemd[1]: Stopping Ceph osd.6 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:08.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:08.413+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:08.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:08.413+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:08.460 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:08.413+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:08.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:08.570+0000 7f76926d9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:57:08.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:08.570+0000 7f76926d9640 -1 osd.6 1062 *** Got signal Terminated *** 2024-09-14T03:57:08.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:08.570+0000 7f76926d9640 -1 osd.6 1062 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:57:08.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:08.699+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:08.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:08.699+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:08.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:08 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:08.699+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:40.514139+0000 front 2024-09-14T03:56:40.514186+0000 (oldest deadline 2024-09-14T03:57:08.114333+0000) 2024-09-14T03:57:09.688 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:09 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:09.436+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:09.688 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:09 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:09.436+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:09.688 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:09 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:09.436+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:09.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:09 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:09.686+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:09.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:09 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:09.686+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:09.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:09 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:09.686+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:40.514139+0000 front 2024-09-14T03:56:40.514186+0000 (oldest deadline 2024-09-14T03:57:08.114333+0000) 2024-09-14T03:57:10.944 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:10.654+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:10.945 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:10.654+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:10.945 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:10.654+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:40.514139+0000 front 2024-09-14T03:56:40.514186+0000 (oldest deadline 2024-09-14T03:57:08.114333+0000) 2024-09-14T03:57:10.945 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:10.469+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:10.945 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:10.469+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:10.945 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:10 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:10.469+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:11.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:11.646+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:11.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:11.646+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:11.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:11.646+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:40.514139+0000 front 2024-09-14T03:56:40.514186+0000 (oldest deadline 2024-09-14T03:57:08.114333+0000) 2024-09-14T03:57:11.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:11.646+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.114717+0000 front 2024-09-14T03:56:48.114703+0000 (oldest deadline 2024-09-14T03:57:11.614703+0000) 2024-09-14T03:57:11.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:11.511+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:11.960 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:11.511+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:11.960 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:11.511+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:11.960 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:11 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:11.511+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:12.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:12 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:12.637+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:23.512299+0000 front 2024-09-14T03:56:23.512402+0000 (oldest deadline 2024-09-14T03:56:48.712768+0000) 2024-09-14T03:57:12.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:12 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:12.637+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:28.713194+0000 front 2024-09-14T03:56:28.713183+0000 (oldest deadline 2024-09-14T03:56:54.613098+0000) 2024-09-14T03:57:12.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:12 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:12.637+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:40.514139+0000 front 2024-09-14T03:56:40.514186+0000 (oldest deadline 2024-09-14T03:57:08.114333+0000) 2024-09-14T03:57:12.959 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:12 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6[96230]: 2024-09-14T03:57:12.637+0000 7f768ace6640 -1 osd.6 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.114717+0000 front 2024-09-14T03:56:48.114703+0000 (oldest deadline 2024-09-14T03:57:11.614703+0000) 2024-09-14T03:57:12.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:12 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:12.504+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:12.960 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:12 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:12.504+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:12.960 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:12 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:12.504+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:12.960 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:12 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:12.504+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:13.941 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:13.526+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:13.941 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:13.526+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:13.941 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:13.526+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:13.941 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:13 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:13.526+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:14.197 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:13 smithi195 podman[113933]: 2024-09-14 03:57:13.940192286 +0000 UTC m=+5.478403672 container died 5305ca81bc266a4ffa06865f4a20ba73c0f10b9f058b99f246872889c8f4b7cf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:14.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:14 smithi195 podman[113933]: 2024-09-14 03:57:14.286843568 +0000 UTC m=+5.825054953 container cleanup 5305ca81bc266a4ffa06865f4a20ba73c0f10b9f058b99f246872889c8f4b7cf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T03:57:14.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:14 smithi195 bash[113933]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6 2024-09-14T03:57:14.895 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:14 smithi195 podman[113948]: 2024-09-14 03:57:14.579771069 +0000 UTC m=+0.639389748 container remove 5305ca81bc266a4ffa06865f4a20ba73c0f10b9f058b99f246872889c8f4b7cf (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-a278231, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:14.895 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:14.486+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:14.896 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:14.486+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:14.896 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:14.486+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:14.896 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:14 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:14.486+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:15.184 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:14 smithi195 podman[114036]: 2024-09-14 03:57:14.797701953 +0000 UTC m=+0.016300759 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:57:15.184 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:14 smithi195 podman[114036]: 2024-09-14 03:57:14.938978977 +0000 UTC m=+0.157577782 container create bac3315682d16add866ef94c430dde2f67d3e8adcd3fa2d38ee66b31a144c83d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:15.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:15 smithi195 podman[114036]: 2024-09-14 03:57:15.183546289 +0000 UTC m=+0.402145094 container init bac3315682d16add866ef94c430dde2f67d3e8adcd3fa2d38ee66b31a144c83d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-14T03:57:15.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:15 smithi195 podman[114036]: 2024-09-14 03:57:15.186906018 +0000 UTC m=+0.405504833 container start bac3315682d16add866ef94c430dde2f67d3e8adcd3fa2d38ee66b31a144c83d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=main-a278231, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-14T03:57:15.459 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:15 smithi195 podman[114036]: 2024-09-14 03:57:15.26669793 +0000 UTC m=+0.485296751 container attach bac3315682d16add866ef94c430dde2f67d3e8adcd3fa2d38ee66b31a144c83d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T03:57:15.460 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 14 03:57:15 smithi195 podman[114036]: 2024-09-14 03:57:15.374050889 +0000 UTC m=+0.592649708 container died bac3315682d16add866ef94c430dde2f67d3e8adcd3fa2d38ee66b31a144c83d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:57:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:15.231Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.110:8765: connect: connection refused" 2024-09-14T03:57:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:15.233Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.110:8765: connect: connection refused" 2024-09-14T03:57:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:15.233Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.110:8765: connect: connection refused" 2024-09-14T03:57:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:15.240Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.110:8765: connect: connection refused" 2024-09-14T03:57:15.460 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:15.241Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.110:8765: connect: connection refused" 2024-09-14T03:57:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:15.242Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.110:8765: connect: connection refused" 2024-09-14T03:57:15.461 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:15.242Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.110:8765: connect: connection refused" 2024-09-14T03:57:15.921 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:15.504+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:15.921 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:15.504+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:15.921 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:15.504+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:15.921 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:15 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:15.504+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:15.945 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.6.service' 2024-09-14T03:57:15.985 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:57:15.985 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-14T03:57:15.985 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-14T03:57:15.986 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.7 2024-09-14T03:57:16.208 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:16 smithi195 systemd[1]: Stopping Ceph osd.7 for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:16.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:16.323+0000 7f1a6a2f2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:57:16.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:16.323+0000 7f1a6a2f2640 -1 osd.7 1062 *** Got signal Terminated *** 2024-09-14T03:57:16.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:16.323+0000 7f1a6a2f2640 -1 osd.7 1062 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:57:16.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:16.486+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:16.709 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:16.486+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:16.710 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:16.486+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:16.710 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:16 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:16.486+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:17.881 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:17 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:17.507+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:17.881 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:17 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:17.507+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:17.881 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:17 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:17.507+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:17.881 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:17 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:17.507+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:18.827 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:18 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:18.512+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:18.827 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:18 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:18.512+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:18.827 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:18 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:18.512+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:18.827 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:18 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:18.512+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:19.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:19 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:19.488+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:19.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:19 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:19.488+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:19.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:19 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:19.488+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:19.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:19 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:19.488+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:20.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:20.514+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2024-09-14T03:56:20.680806+0000 front 2024-09-14T03:56:20.680933+0000 (oldest deadline 2024-09-14T03:56:46.580785+0000) 2024-09-14T03:57:20.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:20.514+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2024-09-14T03:56:31.281928+0000 front 2024-09-14T03:56:31.281811+0000 (oldest deadline 2024-09-14T03:56:54.181739+0000) 2024-09-14T03:57:20.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:20.514+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2024-09-14T03:56:41.683470+0000 front 2024-09-14T03:56:41.683417+0000 (oldest deadline 2024-09-14T03:57:06.383263+0000) 2024-09-14T03:57:20.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:20 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7[100957]: 2024-09-14T03:57:20.514+0000 7f1a628ff640 -1 osd.7 1062 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2024-09-14T03:56:48.084133+0000 front 2024-09-14T03:56:48.084146+0000 (oldest deadline 2024-09-14T03:57:10.984067+0000) 2024-09-14T03:57:21.958 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:21 smithi195 podman[114138]: 2024-09-14 03:57:21.705974704 +0000 UTC m=+5.499111435 container died 98e6e5252ce4da1fbeae53845b24bd2038c1f25dc360b89413492a75a7d3ba1e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-14T03:57:22.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:22 smithi195 podman[114138]: 2024-09-14 03:57:22.039639366 +0000 UTC m=+5.832776096 container cleanup 98e6e5252ce4da1fbeae53845b24bd2038c1f25dc360b89413492a75a7d3ba1e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:57:22.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:22 smithi195 bash[114138]: ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7 2024-09-14T03:57:22.459 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:22 smithi195 podman[114166]: 2024-09-14 03:57:22.224233721 +0000 UTC m=+0.515883349 container remove 98e6e5252ce4da1fbeae53845b24bd2038c1f25dc360b89413492a75a7d3ba1e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-14T03:57:22.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:22 smithi195 podman[114257]: 2024-09-14 03:57:22.551703342 +0000 UTC m=+0.016265522 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:57:22.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:22 smithi195 podman[114257]: 2024-09-14 03:57:22.658379825 +0000 UTC m=+0.122942040 container create 135606c4d9681fec5a9cb489945d3aa481f5970238e58e38244d0162bf3c3b1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-14T03:57:22.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:22 smithi195 podman[114257]: 2024-09-14 03:57:22.893955898 +0000 UTC m=+0.358518063 container init 135606c4d9681fec5a9cb489945d3aa481f5970238e58e38244d0162bf3c3b1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-14T03:57:22.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:22 smithi195 podman[114257]: 2024-09-14 03:57:22.897203313 +0000 UTC m=+0.361765481 container start 135606c4d9681fec5a9cb489945d3aa481f5970238e58e38244d0162bf3c3b1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-14T03:57:23.406 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:22 smithi195 podman[114257]: 2024-09-14 03:57:22.977274935 +0000 UTC m=+0.441837104 container attach 135606c4d9681fec5a9cb489945d3aa481f5970238e58e38244d0162bf3c3b1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T03:57:23.406 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 14 03:57:23 smithi195 podman[114257]: 2024-09-14 03:57:23.092463926 +0000 UTC m=+0.557026102 container died 135606c4d9681fec5a9cb489945d3aa481f5970238e58e38244d0162bf3c3b1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T03:57:23.719 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@osd.7.service' 2024-09-14T03:57:23.758 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:57:23.759 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-14T03:57:23.759 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-09-14T03:57:23.759 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.a 2024-09-14T03:57:24.062 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:23 smithi110 systemd[1]: Stopping Ceph mds.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:24.313 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[111676]: 2024-09-14T03:57:24.060+0000 7f3bec858640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:57:24.314 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:24 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a[111676]: 2024-09-14T03:57:24.060+0000 7f3bec858640 -1 mds.a *** got signal Terminated *** 2024-09-14T03:57:27.196 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:26 smithi110 podman[121398]: 2024-09-14 03:57:26.877218252 +0000 UTC m=+2.918559308 container died ddd26c7ac6c28264ba316d5148cfda05d9bc868edc0cafc3ce03b0b015368bd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.build-date=20240731) 2024-09-14T03:57:27.503 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:27 smithi110 podman[121398]: 2024-09-14 03:57:27.318384899 +0000 UTC m=+3.359725955 container cleanup ddd26c7ac6c28264ba316d5148cfda05d9bc868edc0cafc3ce03b0b015368bd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T03:57:27.504 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:27 smithi110 bash[121398]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a 2024-09-14T03:57:27.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.a.service' 2024-09-14T03:57:27.763 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:27 smithi110 podman[121425]: 2024-09-14 03:57:27.502813343 +0000 UTC m=+0.621316595 container remove ddd26c7ac6c28264ba316d5148cfda05d9bc868edc0cafc3ce03b0b015368bd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:57:27.764 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:27 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.a.service: Deactivated successfully. 2024-09-14T03:57:27.764 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:27 smithi110 systemd[1]: Stopped Ceph mds.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:57:27.764 INFO:journalctl@ceph.mds.a.smithi110.stdout:Sep 14 03:57:27 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.a.service: Consumed 1.384s CPU time. 2024-09-14T03:57:27.777 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:57:27.777 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-09-14T03:57:27.778 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-09-14T03:57:27.778 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.b 2024-09-14T03:57:28.076 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:57:27 smithi195 systemd[1]: Stopping Ceph mds.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:28.346 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:57:28 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b[105203]: 2024-09-14T03:57:28.073+0000 7fa321b8c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:57:28.347 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:57:28 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b[105203]: 2024-09-14T03:57:28.073+0000 7fa321b8c640 -1 mds.b *** got signal Terminated *** 2024-09-14T03:57:32.351 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:57:32 smithi195 podman[114362]: 2024-09-14 03:57:32.08698514 +0000 UTC m=+4.109029153 container died 08047cadae86b419420482ec73ad1e4951e5a2f3a5968d7b5930a07d1b8b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231) 2024-09-14T03:57:32.676 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:57:32 smithi195 podman[114362]: 2024-09-14 03:57:32.432801202 +0000 UTC m=+4.454845214 container cleanup 08047cadae86b419420482ec73ad1e4951e5a2f3a5968d7b5930a07d1b8b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-14T03:57:32.676 INFO:journalctl@ceph.mds.b.smithi195.stdout:Sep 14 03:57:32 smithi195 bash[114362]: ceph-5517b266-7249-11ef-bceb-c7b262605968-mds-b 2024-09-14T03:57:32.821 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@mds.b.service' 2024-09-14T03:57:32.855 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:57:32.855 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-09-14T03:57:32.855 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-14T03:57:32.855 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a 2024-09-14T03:57:33.211 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:32 smithi195 systemd[1]: Stopping Ceph prometheus.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:33.212 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.170Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:57:33.212 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.170Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:57:33.212 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.170Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:57:33.212 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.170Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:57:33.212 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.170Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:57:33.212 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.170Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-14T03:57:33.213 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.170Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:57:33.213 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.170Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-14T03:57:33.213 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.171Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-14T03:57:33.213 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.172Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:57:33.213 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.172Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-14T03:57:33.213 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a[74314]: ts=2024-09-14T03:57:33.172Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-14T03:57:33.213 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 podman[114486]: 2024-09-14 03:57:33.180842616 +0000 UTC m=+0.111814594 container died e0fe3bbc1c08b6e4d30a0d40d2270143f4113effa04a84fb9604d01aab12b1a7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:33.696 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:33 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:57:33.248Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephMgrPrometheusModuleInactive\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.110:8443: connect: connection refused" 2024-09-14T03:57:33.696 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:33 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:57:33.248Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephMgrPrometheusModuleInactive\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-09-14T03:57:33.797 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 podman[114486]: 2024-09-14 03:57:33.57892393 +0000 UTC m=+0.509895905 container cleanup e0fe3bbc1c08b6e4d30a0d40d2270143f4113effa04a84fb9604d01aab12b1a7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:33.797 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Sep 14 03:57:33 smithi195 bash[114486]: ceph-5517b266-7249-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:57:33.941 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5517b266-7249-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-14T03:57:33.974 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T03:57:33.974 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-14T03:57:33.974 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5517b266-7249-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-14T03:57:35.875 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:35 smithi110 systemd[1]: Stopping Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:36.196 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:35 smithi110 ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a[77786]: ts=2024-09-14T03:57:35.873Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:57:36.196 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:35 smithi110 podman[121851]: 2024-09-14 03:57:35.914788025 +0000 UTC m=+0.139558732 container died 02afc2b875a4e86e3e7d582969888ef499c384223706a4408acb467431fa6003 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:36.567 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:36 smithi110 podman[121851]: 2024-09-14 03:57:36.290635843 +0000 UTC m=+0.515406550 container cleanup 02afc2b875a4e86e3e7d582969888ef499c384223706a4408acb467431fa6003 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:36.567 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:36 smithi110 bash[121851]: ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:57:36.854 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:36 smithi110 podman[121862]: 2024-09-14 03:57:36.566780287 +0000 UTC m=+0.643413744 container remove 02afc2b875a4e86e3e7d582969888ef499c384223706a4408acb467431fa6003 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:36.854 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:36 smithi110 podman[121862]: 2024-09-14 03:57:36.650692732 +0000 UTC m=+0.727326149 volume remove 37e69e81b5357465d40bcac9d03dc22b7570e99c74a10859c0a5a3fec87d0a8c 2024-09-14T03:57:37.196 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 systemd[1]: Stopping Ceph node-exporter.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:37.196 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:36 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:57:37.197 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:36 smithi110 systemd[1]: Stopped Ceph alertmanager.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:57:37.197 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 14 03:57:36 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.594s CPU time. 2024-09-14T03:57:37.595 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 podman[121972]: 2024-09-14 03:57:37.297858655 +0000 UTC m=+0.106934173 container died 1b0d2ce16092442a3deff043edfd5a3285fbc29a29b98863f3f2d3fa2da08b7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:37.596 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 podman[121972]: 2024-09-14 03:57:37.595316877 +0000 UTC m=+0.404392392 container cleanup 1b0d2ce16092442a3deff043edfd5a3285fbc29a29b98863f3f2d3fa2da08b7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:37.926 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 bash[121972]: ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a 2024-09-14T03:57:37.926 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 podman[121983]: 2024-09-14 03:57:37.805049541 +0000 UTC m=+0.510850292 container remove 1b0d2ce16092442a3deff043edfd5a3285fbc29a29b98863f3f2d3fa2da08b7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:37.926 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:57:38.196 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-14T03:57:38.196 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 systemd[1]: Stopped Ceph node-exporter.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:57:38.196 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 14 03:57:37 smithi110 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 3.359s CPU time. 2024-09-14T03:57:38.557 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5517b266-7249-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-14T03:57:40.452 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 systemd[1]: Stopping Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:40.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=server t=2024-09-14T03:57:40.450980903Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-14T03:57:40.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=ticker t=2024-09-14T03:57:40.451124914Z level=info msg=stopped last_tick=2024-09-14T03:57:40Z 2024-09-14T03:57:40.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=tracing t=2024-09-14T03:57:40.451152289Z level=info msg="Closing tracing" 2024-09-14T03:57:40.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=grafana-apiserver t=2024-09-14T03:57:40.451177412Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-14T03:57:40.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a[108859]: logger=sqlstore.transactions t=2024-09-14T03:57:40.462624699Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:57:40.710 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 podman[114923]: 2024-09-14 03:57:40.47166609 +0000 UTC m=+0.115429975 container died c9fbdb64fe960ef2581c07b6668b171dac3f11436e566d85b3b5f5a999e16b6a (image=quay.io/ceph/grafana:10.4.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:57:40.985 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 podman[114923]: 2024-09-14 03:57:40.80890694 +0000 UTC m=+0.452670808 container cleanup c9fbdb64fe960ef2581c07b6668b171dac3f11436e566d85b3b5f5a999e16b6a (image=quay.io/ceph/grafana:10.4.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:57:40.986 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 bash[114923]: ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a 2024-09-14T03:57:41.238 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:40 smithi195 podman[114935]: 2024-09-14 03:57:40.984999851 +0000 UTC m=+0.507816886 container remove c9fbdb64fe960ef2581c07b6668b171dac3f11436e566d85b3b5f5a999e16b6a (image=quay.io/ceph/grafana:10.4.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:57:41.238 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:41 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-14T03:57:41.238 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:41 smithi195 systemd[1]: Stopped Ceph grafana.a for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:57:41.238 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Sep 14 03:57:41 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@grafana.a.service: Consumed 7.192s CPU time. 2024-09-14T03:57:41.649 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:41 smithi195 systemd[1]: Stopping Ceph node-exporter.b for 5517b266-7249-11ef-bceb-c7b262605968... 2024-09-14T03:57:41.906 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:41 smithi195 podman[115046]: 2024-09-14 03:57:41.648756788 +0000 UTC m=+0.112097566 container died e217e492de5d7738c7f7962a51fed0d5d79d17c99436cc4f3a142ffdb6eae0db (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:42.208 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:42 smithi195 podman[115046]: 2024-09-14 03:57:42.004793196 +0000 UTC m=+0.468133971 container cleanup e217e492de5d7738c7f7962a51fed0d5d79d17c99436cc4f3a142ffdb6eae0db (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:42.209 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:42 smithi195 bash[115046]: ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b 2024-09-14T03:57:42.570 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:42 smithi195 podman[115058]: 2024-09-14 03:57:42.306107245 +0000 UTC m=+0.659798304 container remove e217e492de5d7738c7f7962a51fed0d5d79d17c99436cc4f3a142ffdb6eae0db (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5517b266-7249-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:42.570 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:42 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:57:42.570 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:42 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-14T03:57:42.570 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:42 smithi195 systemd[1]: Stopped Ceph node-exporter.b for 5517b266-7249-11ef-bceb-c7b262605968. 2024-09-14T03:57:42.570 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Sep 14 03:57:42 smithi195 systemd[1]: ceph-5517b266-7249-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 3.342s CPU time. 2024-09-14T03:57:43.055 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:57:43.088 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:57:43.115 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-14T03:57:43.117 DEBUG:teuthology.misc:Transferring archived files from smithi110:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521/remote/smithi110/crash 2024-09-14T03:57:43.118 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/crash -- . 2024-09-14T03:57:43.251 INFO:teuthology.orchestra.run.smithi110.stderr:tar: /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-14T03:57:43.252 INFO:teuthology.orchestra.run.smithi110.stderr:tar: Error is not recoverable: exiting now 2024-09-14T03:57:43.253 DEBUG:teuthology.misc:Transferring archived files from smithi195:/var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521/remote/smithi195/crash 2024-09-14T03:57:43.254 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/crash -- . 2024-09-14T03:57:43.288 INFO:teuthology.orchestra.run.smithi195.stderr:tar: /var/lib/ceph/5517b266-7249-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-14T03:57:43.288 INFO:teuthology.orchestra.run.smithi195.stderr:tar: Error is not recoverable: exiting now 2024-09-14T03:57:43.289 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-14T03:57:43.289 DEBUG:teuthology.orchestra.run.smithi110:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-14T03:57:43.324 INFO:teuthology.orchestra.run.smithi110.stdout:2024-09-14T03:45:41.765181+0000 mon.a (mon.0) 654 : cluster [WRN] Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-14T03:57:43.325 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-14T03:57:43.325 INFO:tasks.cephadm:Compressing logs... 2024-09-14T03:57:43.325 DEBUG:teuthology.orchestra.run.smithi110:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T03:57:43.368 DEBUG:teuthology.orchestra.run.smithi195:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T03:57:43.393 INFO:teuthology.orchestra.run.smithi110.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-14T03:57:43.393 INFO:teuthology.orchestra.run.smithi110.stderr:: No such file or directory 2024-09-14T03:57:43.393 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-14T03:57:43.394 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-14T03:57:43.399 INFO:teuthology.orchestra.run.smithi195.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-14T03:57:43.399 INFO:teuthology.orchestra.run.smithi195.stderr:: No such file or directory 2024-09-14T03:57:43.399 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-14T03:57:43.399 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mds.b.log 2024-09-14T03:57:43.399 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-14T03:57:43.399 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-14T03:57:43.400 INFO:teuthology.orchestra.run.smithi195.stderr: 83.6% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mds.b.log.gz 2024-09-14T03:57:43.400 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-14T03:57:43.400 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.4.log: /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-14T03:57:43.400 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-14T03:57:43.400 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.log 2024-09-14T03:57:43.400 INFO:teuthology.orchestra.run.smithi195.stderr: 84.2% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-14T03:57:43.400 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-14T03:57:43.401 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-14T03:57:43.401 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mgr.x.log: 90.4% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-14T03:57:43.402 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-14T03:57:43.402 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-14T03:57:43.403 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mds.a.log 2024-09-14T03:57:43.403 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-14T03:57:43.403 INFO:teuthology.orchestra.run.smithi110.stderr:gzip/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mds.a.log: -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-14T03:57:43.403 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-14T03:57:43.403 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.a.log: 91.0% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-14T03:57:43.403 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-14T03:57:43.404 INFO:teuthology.orchestra.run.smithi195.stderr: 85.7% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-14T03:57:43.404 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-14T03:57:43.405 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.2.log: 94.1% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-14T03:57:43.406 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.log 2024-09-14T03:57:43.406 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mgr.y.log: 91.8% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mds.a.log.gz 2024-09-14T03:57:43.415 INFO:teuthology.orchestra.run.smithi110.stderr: 91.1% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-14T03:57:43.417 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.1% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-14T03:57:43.418 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-14T03:57:43.418 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-14T03:57:43.427 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.3.log: 92.0% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-14T03:57:43.428 INFO:teuthology.orchestra.run.smithi195.stderr: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-14T03:57:43.436 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/cephadm.log: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-14T03:57:43.465 INFO:teuthology.orchestra.run.smithi195.stderr: 89.9% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-14T03:57:43.856 INFO:teuthology.orchestra.run.smithi110.stderr: 89.3% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-14T03:57:43.973 INFO:teuthology.orchestra.run.smithi195.stderr: 92.1% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-14T03:57:44.058 INFO:teuthology.orchestra.run.smithi110.stderr: 91.6% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-14T03:57:45.005 INFO:teuthology.orchestra.run.smithi110.stderr: 90.8% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-14T03:57:47.986 INFO:teuthology.orchestra.run.smithi195.stderr: 94.3% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-14T03:57:48.812 INFO:teuthology.orchestra.run.smithi195.stderr: 94.3% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-14T03:57:49.165 INFO:teuthology.orchestra.run.smithi110.stderr: 93.8% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-14T03:57:49.367 INFO:teuthology.orchestra.run.smithi195.stderr: 94.0% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-14T03:57:49.684 INFO:teuthology.orchestra.run.smithi110.stderr: 93.9% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-14T03:57:49.937 INFO:teuthology.orchestra.run.smithi195.stderr: 94.0% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-14T03:57:49.939 INFO:teuthology.orchestra.run.smithi195.stderr: 2024-09-14T03:57:49.939 INFO:teuthology.orchestra.run.smithi195.stderr:real 0m6.556s 2024-09-14T03:57:49.939 INFO:teuthology.orchestra.run.smithi195.stderr:user 0m22.290s 2024-09-14T03:57:49.939 INFO:teuthology.orchestra.run.smithi195.stderr:sys 0m0.913s 2024-09-14T03:57:52.016 INFO:teuthology.orchestra.run.smithi110.stderr: 94.4% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-14T03:57:52.020 INFO:teuthology.orchestra.run.smithi110.stderr: 94.3% -- replaced with /var/log/ceph/5517b266-7249-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-14T03:57:52.022 INFO:teuthology.orchestra.run.smithi110.stderr: 2024-09-14T03:57:52.022 INFO:teuthology.orchestra.run.smithi110.stderr:real 0m8.640s 2024-09-14T03:57:52.022 INFO:teuthology.orchestra.run.smithi110.stderr:user 0m30.943s 2024-09-14T03:57:52.022 INFO:teuthology.orchestra.run.smithi110.stderr:sys 0m1.086s 2024-09-14T03:57:52.023 INFO:tasks.cephadm:Archiving logs... 2024-09-14T03:57:52.023 DEBUG:teuthology.misc:Transferring archived files from smithi110:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521/remote/smithi110/log 2024-09-14T03:57:52.024 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T03:57:56.172 DEBUG:teuthology.misc:Transferring archived files from smithi195:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521/remote/smithi195/log 2024-09-14T03:57:56.173 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T03:57:59.396 INFO:tasks.cephadm:Removing cluster... 2024-09-14T03:57:59.396 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5517b266-7249-11ef-bceb-c7b262605968 --force 2024-09-14T03:57:59.908 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5517b266-7249-11ef-bceb-c7b262605968 --force 2024-09-14T03:58:00.409 INFO:tasks.cephadm:Removing cephadm ... 2024-09-14T03:58:00.410 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-14T03:58:00.426 DEBUG:teuthology.orchestra.run.smithi195:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-14T03:58:00.441 INFO:tasks.cephadm:Teardown complete 2024-09-14T03:58:00.442 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-14T03:58:00.452 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-14T03:58:00.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T03:58:00.468 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T03:58:00.557 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-14T03:58:00.567 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-14T03:58:00.567 DEBUG:teuthology.orchestra.run.smithi110:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:58:00.570 DEBUG:teuthology.orchestra.run.smithi195:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:58:00.582 INFO:teuthology.orchestra.run.smithi110.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:58:00.584 INFO:teuthology.orchestra.run.smithi195.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:58:00.587 INFO:teuthology.orchestra.run.smithi110.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:58:00.587 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================== 2024-09-14T03:58:00.587 INFO:teuthology.orchestra.run.smithi110.stdout:^* hv01.front.sepia.ceph.com 3 6 377 34 +80us[ +134us] +/- 55ms 2024-09-14T03:58:00.587 INFO:teuthology.orchestra.run.smithi110.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 35 -452us[ -398us] +/- 73ms 2024-09-14T03:58:00.587 INFO:teuthology.orchestra.run.smithi110.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 35 +378us[ +432us] +/- 73ms 2024-09-14T03:58:00.588 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:58:00.588 INFO:teuthology.orchestra.run.smithi195.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:58:00.588 INFO:teuthology.orchestra.run.smithi195.stdout:=============================================================================== 2024-09-14T03:58:00.588 INFO:teuthology.orchestra.run.smithi195.stdout:^* hv01.front.sepia.ceph.com 3 6 377 32 +71us[ +70us] +/- 55ms 2024-09-14T03:58:00.588 INFO:teuthology.orchestra.run.smithi195.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 35 -468us[ -470us] +/- 73ms 2024-09-14T03:58:00.589 INFO:teuthology.orchestra.run.smithi195.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 34 +375us[ +374us] +/- 73ms 2024-09-14T03:58:00.589 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:58:00.590 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-14T03:58:00.605 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-14T03:58:00.605 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-14T03:58:00.655 DEBUG:teuthology.orchestra.run.smithi110:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T03:58:00.732 DEBUG:teuthology.orchestra.run.smithi195:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T03:58:00.806 DEBUG:teuthology.orchestra.run.smithi110:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-14T03:58:00.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:58:00.835 DEBUG:teuthology.orchestra.run.smithi195:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-14T03:58:00.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:58:00.864 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-14T03:58:00.875 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-14T03:58:00.896 INFO:teuthology.task.internal:Duration was 2141.143785 seconds 2024-09-14T03:58:00.897 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-14T03:58:00.947 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-14T03:58:00.948 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T03:58:00.950 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T03:58:00.987 INFO:teuthology.orchestra.run.smithi110.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:58:00.988 INFO:teuthology.orchestra.run.smithi195.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:58:01.242 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-14T03:58:01.243 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi110.front.sepia.ceph.com 2024-09-14T03:58:01.243 DEBUG:teuthology.orchestra.run.smithi110:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T03:58:01.294 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi195.front.sepia.ceph.com 2024-09-14T03:58:01.295 DEBUG:teuthology.orchestra.run.smithi195:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T03:58:01.324 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-14T03:58:01.324 DEBUG:teuthology.orchestra.run.smithi110:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T03:58:01.338 DEBUG:teuthology.orchestra.run.smithi195:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T03:58:01.390 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-14T03:58:01.391 DEBUG:teuthology.orchestra.run.smithi110:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T03:58:01.405 DEBUG:teuthology.orchestra.run.smithi195:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T03:58:01.852 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-14T03:58:01.871 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-14T03:58:01.871 DEBUG:teuthology.orchestra.run.smithi110:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T03:58:01.898 DEBUG:teuthology.orchestra.run.smithi195:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T03:58:01.924 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-14T03:58:01.941 DEBUG:teuthology.orchestra.run.smithi110:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T03:58:01.943 DEBUG:teuthology.orchestra.run.smithi195:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T03:58:01.964 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern = core 2024-09-14T03:58:01.987 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern = core 2024-09-14T03:58:02.006 DEBUG:teuthology.orchestra.run.smithi110:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T03:58:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:58:02.033 DEBUG:teuthology.orchestra.run.smithi195:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T03:58:02.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:58:02.062 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-14T03:58:02.072 INFO:teuthology.task.internal:Transferring archived files... 2024-09-14T03:58:02.073 DEBUG:teuthology.misc:Transferring archived files from smithi110:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521/remote/smithi110 2024-09-14T03:58:02.074 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T03:58:02.120 DEBUG:teuthology.misc:Transferring archived files from smithi195:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904521/remote/smithi195 2024-09-14T03:58:02.120 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T03:58:02.162 INFO:teuthology.task.internal:Removing archive directory... 2024-09-14T03:58:02.162 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T03:58:02.165 DEBUG:teuthology.orchestra.run.smithi195:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T03:58:02.208 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-14T03:58:02.221 INFO:teuthology.task.internal:Not uploading archives. 2024-09-14T03:58:02.221 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-14T03:58:02.261 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-14T03:58:02.261 DEBUG:teuthology.orchestra.run.smithi110:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T03:58:02.264 DEBUG:teuthology.orchestra.run.smithi195:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T03:58:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: 262169 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 03:58 /home/ubuntu/cephtest 2024-09-14T03:58:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: 666398 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 03:51 /home/ubuntu/cephtest/mnt.0 2024-09-14T03:58:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: 666399 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 03:55 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T03:58:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: 679878 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 03:55 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:58:02.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:58:02.279 INFO:teuthology.orchestra.run.smithi110.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-14T03:58:02.279 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi110 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-14T03:58:02.288 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-14T03:58:02.307 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-14T03:58:02.358 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi110 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-14T03:58:02.362 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} duration: 2141.1437854766846 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi110 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=2efbdfcab6e64117b08c26dc9412f8bf status: fail success: false 2024-09-14T03:58:02.362 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T03:58:02.372 INFO:teuthology.orchestra.run.smithi195.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 03:58 /home/ubuntu/cephtest 2024-09-14T03:58:02.474 INFO:teuthology.run:FAIL